def get_spark_version(service_name, component_name, yarn_version):
"""
Attempts to calculate the correct version placeholder value for spark or spark2 based on
what is installed in the cluster. If Spark is not installed, then this value will need to be
that of YARN so it can still find the correct shuffle class.
On cluster installs, we have not yet calcualted any versions and all known values could be None.
This doesn't affect daemons, but it does affect client-only hosts where they will never receive
a start command after install. Therefore, this function will attempt to use stack-select as a
last resort to get a value value.
:param service_name: the service name (SPARK, SPARK2, etc)
:param component_name: the component name (SPARK_CLIENT, etc)
:param yarn_version: the default version of Yarn to use if no spark is installed
:return: a value for the version placeholder in shuffle classpath properties
"""
# start off seeing if we need to populate a default value for YARN
if yarn_version is None:
yarn_version = component_version.get_component_repository_version(service_name = "YARN",
component_name = "YARN_CLIENT")
# now try to get the version of spark/spark2, defaulting to the version if YARN
shuffle_classpath_version = component_version.get_component_repository_version(service_name = service_name,
component_name = component_name, default_value = yarn_version)
# even with the default of using YARN's version, on an install this might be None since we haven't
# calculated the version of YARN yet - use stack_select as a last ditch effort
if shuffle_classpath_version is None:
try:
shuffle_classpath_version = stack_select.get_role_component_current_stack_version()
except:
Logger.exception("Unable to query for the correct shuffle classpath")
return shuffle_classpath_version
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
# grab the current version of the component
pre_upgrade_version = stack_select.get_role_component_current_stack_version(
)
if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE,
params.version):
stack_select.select_packages(params.version)
# This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary.
if params.version and params.upgrade_direction:
src_version = dst_version = None
if params.upgrade_direction == Direction.UPGRADE:
src_version = upgrade_summary.get_source_version(
"KAFKA", default_version=params.version)
dst_version = upgrade_summary.get_target_version(
"KAFKA", default_version=params.version)
else:
# These represent the original values during the UPGRADE direction
src_version = upgrade_summary.get_target_version(
"KAFKA", default_version=params.version)
dst_version = upgrade_summary.get_source_version(
"KAFKA", default_version=params.version)
if not check_stack_feature(
StackFeature.KAFKA_ACL_MIGRATION_SUPPORT,
src_version) and check_stack_feature(
StackFeature.KAFKA_ACL_MIGRATION_SUPPORT, dst_version):
# Calling the acl migration script requires the configs to be present.
self.configure(env, upgrade_type=upgrade_type)
upgrade.run_migration(env, upgrade_type)
def get_current_version(service=None, use_upgrading_version_during_upgrade=True):
"""
Get the effective version to use to copy the tarballs to.
:param service: the service name when checking for an upgrade. made optional for unknown \
code bases that may be using this function
:param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet.
:return: Version, or False if an error occurred.
"""
from resource_management.libraries.functions import upgrade_summary
# get the version for this command
version = stack_features.get_stack_feature_version(Script.get_config())
if service is not None:
version = upgrade_summary.get_target_version(service_name=service, default_version=version)
# if there is no upgrade, then use the command's version
if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade:
Logger.info("Tarball version was calcuated as {0}. Use Command Version: {1}".format(
version, use_upgrading_version_during_upgrade))
return version
# we're in an upgrade and we need to use an older version
current_version = stack_select.get_role_component_current_stack_version()
if service is not None:
current_version = upgrade_summary.get_source_version(service_name=service, default_version=current_version)
if current_version is None:
Logger.warning("Unable to determine the current version of the component for this command; unable to copy the tarball")
return False
return current_version;
def get_current_version(use_upgrading_version_during_upgrade=True):
"""
Get the effective version to use to copy the tarballs to.
:param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet.
:return: Version, or False if an error occurred.
"""
# get the version for this command
version = stack_features.get_stack_feature_version(Script.get_config())
# if there is no upgrade, then use the command's version
if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade:
Logger.info(
"Tarball version was calcuated as {0}. Use Command Version: {1}".
format(version, use_upgrading_version_during_upgrade))
return version
# we're in an upgrade and we need to use an older version
current_version = stack_select.get_role_component_current_stack_version()
if current_version is None:
Logger.warning(
"Unable to determine the current version of the component for this command; unable to copy the tarball"
)
return False
return current_version
def get_current_component_version():
"""
Returns best available version for the component at different stages (install, start, stop)
:return best matched version or None
:rtype str|None
"""
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.stack_select import get_role_component_current_stack_version
from resource_management.libraries.functions.repository_util import CommandRepository
version = default("/commandParams/version", None)
if not version:
repository = CommandRepository(default("/repositoryFile", {}))
if not repository.resolved:
try:
version = get_role_component_current_stack_version()
except (Fail, TypeError):
pass
else:
version = repository.version_string
return version
