def get_current_version(service=None, use_upgrading_version_during_upgrade=True):
"""
Get the effective version to use to copy the tarballs to.
:param service: the service name when checking for an upgrade. made optional for unknown \
code bases that may be using this function
:param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet.
:return: Version, or False if an error occurred.
"""
from resource_management.libraries.functions import upgrade_summary
# get the version for this command
version = stack_features.get_stack_feature_version(Script.get_config())
if service is not None:
version = upgrade_summary.get_target_version(service_name=service, default_version=version)
# if there is no upgrade, then use the command's version
if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade:
Logger.info("Tarball version was calcuated as {0}. Use Command Version: {1}".format(
version, use_upgrading_version_during_upgrade))
return version
# we're in an upgrade and we need to use an older version
current_version = stack_select.get_role_component_current_stack_version()
if service is not None:
current_version = upgrade_summary.get_source_version(service_name=service, default_version=current_version)
if current_version is None:
Logger.warning("Unable to determine the current version of the component for this command; unable to copy the tarball")
return False
return current_version;
def get_current_version(use_upgrading_version_during_upgrade=True):
"""
Get the effective version to use to copy the tarballs to.
:param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet.
:return: Version, or False if an error occurred.
"""
# get the version for this command
version = stack_features.get_stack_feature_version(Script.get_config())
# if there is no upgrade, then use the command's version
if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade:
Logger.info(
"Tarball version was calcuated as {0}. Use Command Version: {1}".
format(version, use_upgrading_version_during_upgrade))
return version
# we're in an upgrade and we need to use an older version
current_version = stack_select.get_role_component_current_stack_version()
if current_version is None:
Logger.warning(
"Unable to determine the current version of the component for this command; unable to copy the tarball"
)
return False
return current_version
def get_hadoop_conf_dir(force_latest_on_upgrade=False):
"""
Gets the shared hadoop conf directory using:
1. Start with /etc/hadoop/conf
2. When the stack is greater than HDP-2.2, use /usr/hdp/current/hadoop-client/conf
3. Only when doing a RU and HDP-2.3 or higher, use the value as computed
by conf-select. This is in the form /usr/hdp/VERSION/hadoop/conf to make sure
the configs are written in the correct place. However, if the component itself has
not yet been upgraded, it should use the hadoop configs from the prior version.
This will perform an hdp-select status to determine which version to use.
:param force_latest_on_upgrade: if True, then force the returned path to always
be that of the upgrade target version, even if hdp-select has not been called. This
is primarily used by hooks like before-ANY to ensure that hadoop environment
configurations are written to the correct location since they are written out
before the hdp-select/conf-select would have been called.
"""
hadoop_conf_dir = "/etc/hadoop/conf"
stack_name = None
version = None
allow_setting_conf_select_symlink = False
if not Script.in_stack_upgrade():
# During normal operation, the HDP stack must be 2.3 or higher
if Script.is_hdp_stack_greater_or_equal("2.2"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
if Script.is_hdp_stack_greater_or_equal("2.3"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
stack_name = default("/hostLevelParams/stack_name", None)
version = default("/commandParams/version", None)
if stack_name and version:
version = str(version)
allow_setting_conf_select_symlink = True
else:
# During an upgrade/downgrade, which can be a Rolling or Express Upgrade, need to calculate it based on the version
'''
Whenever upgrading to HDP 2.2, or downgrading back to 2.2, need to use /etc/hadoop/conf
Whenever upgrading to HDP 2.3, or downgrading back to 2.3, need to use a versioned hadoop conf dir
Type__|_Source_|_Target_|_Direction_____________|_Comment_____________________________________________________________
Normal| | 2.2 | | Use /etc/hadoop/conf
Normal| | 2.3 | | Use /etc/hadoop/conf, which should be a symlink to /usr/hdp/current/hadoop-client/conf
EU | 2.1 | 2.3 | Upgrade | Use versioned /usr/hdp/current/hadoop-client/conf
| | | No Downgrade Allowed | Invalid
EU/RU | 2.2 | 2.2.* | Any | Use /usr/hdp/current/hadoop-client/conf
EU/RU | 2.2 | 2.3 | Upgrade | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
| | | Downgrade | Use /usr/hdp/current/hadoop-client/conf
EU/RU | 2.3 | 2.3.* | Any | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
'''
# The method "is_hdp_stack_greater_or_equal" uses "stack_version" which is the desired stack, e.g., 2.2 or 2.3
# In an RU, it is always the desired stack, and doesn't change even during the Downgrade!
# In an RU Downgrade from HDP 2.3 to 2.2, the first thing we do is
# rm /etc/[component]/conf and then mv /etc/[component]/conf.backup /etc/[component]/conf
if Script.is_hdp_stack_greater_or_equal("2.2"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
# This contains the "version", including the build number, that is actually used during a stack upgrade and
# is the version upgrading/downgrading to.
stack_info = hdp_select._get_upgrade_stack()
if stack_info is not None:
stack_name = stack_info[0]
version = stack_info[1]
else:
raise Fail("Unable to get parameter 'version'")
Logger.info(
"In the middle of a stack upgrade/downgrade for Stack {0} and destination version {1}, determining which hadoop conf dir to use."
.format(stack_name, version))
# This is the version either upgrading or downgrading to.
if compare_versions(format_hdp_stack_version(version),
"2.3.0.0") >= 0:
# Determine if hdp-select has been run and if not, then use the current
# hdp version until this component is upgraded.
if not force_latest_on_upgrade:
current_hdp_version = hdp_select.get_role_component_current_hdp_version(
)
if current_hdp_version is not None and version != current_hdp_version:
version = current_hdp_version
Logger.info(
"hdp-select has not yet been called to update the symlink for this component, keep using version {0}"
.format(current_hdp_version))
# Only change the hadoop_conf_dir path, don't conf-select this older version
hadoop_conf_dir = "/usr/hdp/{0}/hadoop/conf".format(version)
Logger.info("Hadoop conf dir: {0}".format(hadoop_conf_dir))
allow_setting_conf_select_symlink = True
if allow_setting_conf_select_symlink:
# If not in the middle of an upgrade and on HDP 2.3 or higher, or if
# upgrading stack to version 2.3.0.0 or higher (which may be upgrade or downgrade), then consider setting the
# symlink for /etc/hadoop/conf.
# If a host does not have any HDFS or YARN components (e.g., only ZK), then it will not contain /etc/hadoop/conf
# Therefore, any calls to conf-select will fail.
# For that reason, if the hadoop conf directory exists, then make sure it is set.
if os.path.exists(hadoop_conf_dir):
Logger.info(
"The hadoop conf dir {0} exists, will call conf-select on it for version {1}"
.format(hadoop_conf_dir, version))
select(stack_name, "hadoop", version)
Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))
return hadoop_conf_dir
def get_hadoop_conf_dir(force_latest_on_upgrade=False):
"""
Gets the shared hadoop conf directory using:
1. Start with /etc/hadoop/conf
2. When the stack is greater than HDP-2.2, use /usr/hdp/current/hadoop-client/conf
3. Only when doing a RU and HDP-2.3 or higher, use the value as computed
by conf-select. This is in the form /usr/hdp/VERSION/hadoop/conf to make sure
the configs are written in the correct place. However, if the component itself has
not yet been upgraded, it should use the hadoop configs from the prior version.
This will perform an hdp-select status to determine which version to use.
:param force_latest_on_upgrade: if True, then force the returned path to always
be that of the upgrade target version, even if hdp-select has not been called. This
is primarily used by hooks like before-ANY to ensure that hadoop environment
configurations are written to the correct location since they are written out
before the hdp-select/conf-select would have been called.
"""
hadoop_conf_dir = "/etc/hadoop/conf"
stack_name = None
version = None
allow_setting_conf_select_symlink = False
if not Script.in_stack_upgrade():
# During normal operation, the HDP stack must be 2.3 or higher
if Script.is_hdp_stack_greater_or_equal("2.2"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
if Script.is_hdp_stack_greater_or_equal("2.3"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
stack_name = default("/hostLevelParams/stack_name", None)
version = default("/commandParams/version", None)
if stack_name and version:
version = str(version)
allow_setting_conf_select_symlink = True
else:
# During an upgrade/downgrade, which can be a Rolling or Express Upgrade, need to calculate it based on the version
'''
Whenever upgrading to HDP 2.2, or downgrading back to 2.2, need to use /etc/hadoop/conf
Whenever upgrading to HDP 2.3, or downgrading back to 2.3, need to use a versioned hadoop conf dir
Type__|_Source_|_Target_|_Direction_____________|_Comment_____________________________________________________________
Normal| | 2.2 | | Use /etc/hadoop/conf
Normal| | 2.3 | | Use /etc/hadoop/conf, which should be a symlink to /usr/hdp/current/hadoop-client/conf
EU | 2.1 | 2.3 | Upgrade | Use versioned /usr/hdp/current/hadoop-client/conf
| | | No Downgrade Allowed | Invalid
EU/RU | 2.2 | 2.2.* | Any | Use /usr/hdp/current/hadoop-client/conf
EU/RU | 2.2 | 2.3 | Upgrade | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
| | | Downgrade | Use /usr/hdp/current/hadoop-client/conf
EU/RU | 2.3 | 2.3.* | Any | Use /usr/hdp/$version/hadoop/conf, which should be a symlink destination
'''
# The method "is_hdp_stack_greater_or_equal" uses "stack_version" which is the desired stack, e.g., 2.2 or 2.3
# In an RU, it is always the desired stack, and doesn't change even during the Downgrade!
# In an RU Downgrade from HDP 2.3 to 2.2, the first thing we do is
# rm /etc/[component]/conf and then mv /etc/[component]/conf.backup /etc/[component]/conf
if Script.is_hdp_stack_greater_or_equal("2.2"):
hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
# This contains the "version", including the build number, that is actually used during a stack upgrade and
# is the version upgrading/downgrading to.
stack_info = hdp_select._get_upgrade_stack()
if stack_info is not None:
stack_name = stack_info[0]
version = stack_info[1]
else:
raise Fail("Unable to get parameter 'version'")
Logger.info("In the middle of a stack upgrade/downgrade for Stack {0} and destination version {1}, determining which hadoop conf dir to use.".format(stack_name, version))
# This is the version either upgrading or downgrading to.
if compare_versions(format_hdp_stack_version(version), "2.3.0.0") >= 0:
# Determine if hdp-select has been run and if not, then use the current
# hdp version until this component is upgraded.
if not force_latest_on_upgrade:
current_hdp_version = hdp_select.get_role_component_current_hdp_version()
if current_hdp_version is not None and version != current_hdp_version:
version = current_hdp_version
Logger.info("hdp-select has not yet been called to update the symlink for this component, keep using version {0}".format(current_hdp_version))
# Only change the hadoop_conf_dir path, don't conf-select this older version
hadoop_conf_dir = "/usr/hdp/{0}/hadoop/conf".format(version)
Logger.info("Hadoop conf dir: {0}".format(hadoop_conf_dir))
allow_setting_conf_select_symlink = True
if allow_setting_conf_select_symlink:
# If not in the middle of an upgrade and on HDP 2.3 or higher, or if
# upgrading stack to version 2.3.0.0 or higher (which may be upgrade or downgrade), then consider setting the
# symlink for /etc/hadoop/conf.
# If a host does not have any HDFS or YARN components (e.g., only ZK), then it will not contain /etc/hadoop/conf
# Therefore, any calls to conf-select will fail.
# For that reason, if the hadoop conf directory exists, then make sure it is set.
if os.path.exists(hadoop_conf_dir):
Logger.info("The hadoop conf dir {0} exists, will call conf-select on it for version {1}".format(hadoop_conf_dir, version))
select(stack_name, "hadoop", version)
Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))
return hadoop_conf_dir
def get_hadoop_conf_dir(force_latest_on_upgrade=False):
"""
Gets the shared hadoop conf directory using:
1. Start with /etc/hadoop/conf
2. When the stack is greater than HDP-2.2, use <stack-root>/current/hadoop-client/conf
3. Only when doing a RU and HDP-2.3 or higher, use the value as computed
by <conf-selector-tool>. This is in the form <stack-root>/VERSION/hadoop/conf to make sure
the configs are written in the correct place. However, if the component itself has
not yet been upgraded, it should use the hadoop configs from the prior version.
This will perform an <stack-selector-tool> status to determine which version to use.
:param force_latest_on_upgrade: if True, then force the returned path to always
be that of the upgrade target version, even if <stack-selector-tool> has not been called. This
is primarily used by hooks like before-ANY to ensure that hadoop environment
configurations are written to the correct location since they are written out
before the <stack-selector-tool>/<conf-selector-tool> would have been called.
"""
hadoop_conf_dir = "/etc/hadoop/conf"
stack_name = None
stack_root = Script.get_stack_root()
stack_version = Script.get_stack_version()
version = None
if not Script.in_stack_upgrade():
# During normal operation, the HDP stack must be 2.3 or higher
if stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
stack_version):
hadoop_conf_dir = os.path.join(stack_root, "current",
"hadoop-client", "conf")
if stack_version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, stack_version):
hadoop_conf_dir = os.path.join(stack_root, "current",
"hadoop-client", "conf")
stack_name = default("/hostLevelParams/stack_name", None)
version = default("/commandParams/version", None)
if not os.path.islink(hadoop_conf_dir) and stack_name and version:
version = str(version)
else:
# The "stack_version" is the desired stack, e.g., 2.2 or 2.3
# In an RU, it is always the desired stack, and doesn't change even during the Downgrade!
# In an RU Downgrade from HDP 2.3 to 2.2, the first thing we do is
# rm /etc/[component]/conf and then mv /etc/[component]/conf.backup /etc/[component]/conf
if stack_version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
stack_version):
hadoop_conf_dir = os.path.join(stack_root, "current",
"hadoop-client", "conf")
# This contains the "version", including the build number, that is actually used during a stack upgrade and
# is the version upgrading/downgrading to.
stack_info = stack_select._get_upgrade_stack()
if stack_info is None:
raise Fail(
"Unable to retrieve the upgrade/downgrade stack information from the request"
)
stack_name = stack_info[0]
version = stack_info[1]
Logger.info(
"An upgrade/downgrade for {0}-{1} is in progress, determining which hadoop conf dir to use."
.format(stack_name, version))
# This is the version either upgrading or downgrading to.
if version and check_stack_feature(StackFeature.CONFIG_VERSIONING,
version):
# Determine if <stack-selector-tool> has been run and if not, then use the current
# hdp version until this component is upgraded.
if not force_latest_on_upgrade:
current_stack_version = stack_select.get_role_component_current_stack_version(
)
if current_stack_version is not None and version != current_stack_version:
version = current_stack_version
stack_selector_name = stack_tools.get_stack_tool_name(
stack_tools.STACK_SELECTOR_NAME)
Logger.info(
"{0} has not yet been called to update the symlink for this component, "
"keep using version {1}".format(
stack_selector_name, current_stack_version))
# Only change the hadoop_conf_dir path, don't <conf-selector-tool> this older version
hadoop_conf_dir = os.path.join(stack_root, version, "hadoop",
"conf")
Logger.info("Hadoop conf dir: {0}".format(hadoop_conf_dir))
Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))
return hadoop_conf_dir
