def check_partial_install(self):
"""
If an installation did not complete successfully, check if installation was partially complete and
log the partially completed version to REPO_VERSION_HISTORY_FILE.
:return:
"""
Logger.info(
"Installation of packages failed. Checking if installation was partially complete"
)
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
# Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
partial_install_version = next(iter(deltas)).strip()
write_actual_version_to_history_file(normalized_repo_version,
partial_install_version)
Logger.info("Version {0} was partially installed. ".format(
partial_install_version))
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is, in order to save it.
"""
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions()
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get HDP version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
else:
Logger.info(
"Cannot determine a new actual version installed by using the delta method."
)
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, it is important to search the
# repo version history file to determine if we previously did write an actual_version.
self.actual_version = read_actual_version_from_history_file(
normalized_repo_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by parsing file {1}".format(
self.actual_version, REPO_VERSION_HISTORY_FILE))
elif self.repo_version_with_build_number is None:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is, in order to save it.
"""
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions()
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get HDP version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
else:
Logger.info(
"Cannot determine a new actual version installed by using the delta method."
)
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, it is important to search the
# repo version history file to determine if we previously did write an actual_version.
self.actual_version = read_actual_version_from_history_file(
normalized_repo_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by parsing file {1}".format(
self.actual_version, REPO_VERSION_HISTORY_FILE))
elif self.repo_version_with_build_number is None:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def check_partial_install(self):
"""
If an installation did not complete successfully, check if installation was partially complete and
log the partially completed version to REPO_VERSION_HISTORY_FILE.
:return:
"""
Logger.info("Installation of packages failed. Checking if installation was partially complete")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
# Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
partial_install_version = next(iter(deltas)).strip()
write_actual_version_to_history_file(normalized_repo_version, partial_install_version)
Logger.info("Version {0} was partially installed. ".format(partial_install_version))
class InstallPackages(Script):
"""
This script is a part of Rolling Upgrade workflow and is described at
appropriate design doc.
It installs repositories to the node and then installs packages.
For now, repositories are installed into individual files.
"""
UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
REPO_FILE_NAME_PREFIX = 'HDP-'
STACK_TO_ROOT_FOLDER = {"HDP": "/usr/hdp"}
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
# Select dict that contains parameters
try:
self.repository_version = config['roleParams'][
'repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
# Last try
self.repository_version = config['commandParams'][
'repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
# current stack information
self.current_hdp_stack_version = None
if 'stack_version' in config['hostLevelParams']:
current_stack_version_unformatted = str(
config['hostLevelParams']['stack_version'])
self.current_hdp_stack_version = format_hdp_stack_version(
current_stack_version_unformatted)
stack_name = None
self.stack_root_folder = None
if stack_id and "-" in stack_id:
stack_split = stack_id.split("-")
if len(stack_split) == 2:
stack_name = stack_split[0].upper()
if stack_name in self.STACK_TO_ROOT_FOLDER:
self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[
stack_name]
if self.stack_root_folder is None:
raise Fail(
"Cannot determine the stack's root directory by parsing the stack_id property, {0}"
.format(str(stack_id)))
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(
self.repository_version))
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(
url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos()
except Exception, err:
Logger.logger.exception(
"Cannot distribute repositories. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'ambari_repositories': installed_repositories,
'installed_repository_version': self.repository_version,
'stack_id': stack_id,
'package_installation_result': 'FAIL'
}
self.put_structured_out(self.structured_output)
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_hdp_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception, err:
num_errors += 1
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output[
'actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages"
.format(self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info(
"Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info(
"Will try to find for the actual version by searching for best possible match in the list of versions installed"
)
self.actual_version = self.find_best_fit_version(
new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by searching for best possible match"
.format(self.actual_version))
else:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output[
'actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages"
.format(self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info(
"Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info(
"Will try to find for the actual version by searching for best possible match in the list of versions installed"
)
self.actual_version = self.find_best_fit_version(
new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by searching for best possible match"
.format(self.actual_version))
else:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
class InstallPackages(Script):
"""
This script is a part of Rolling Upgrade workflow and is described at
appropriate design doc.
It installs repositories to the node and then installs packages.
For now, repositories are installed into individual files.
"""
UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
REPO_FILE_NAME_PREFIX = 'HDP-'
STACK_TO_ROOT_FOLDER = {"HDP": "/usr/hdp"}
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
# Select dict that contains parameters
try:
self.repository_version = config['roleParams'][
'repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
# Last try
self.repository_version = config['commandParams'][
'repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
stack_name = None
self.stack_root_folder = None
if stack_id and "-" in stack_id:
stack_split = stack_id.split("-")
if len(stack_split) == 2:
stack_name = stack_split[0].upper()
if stack_name in self.STACK_TO_ROOT_FOLDER:
self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[
stack_name]
if self.stack_root_folder is None:
raise Fail(
"Cannot determine the stack's root directory by parsing the stack_id property, {0}"
.format(str(stack_id)))
self.repository_version = self.repository_version.strip()
# Install/update repositories
installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
Logger.info("Will install packages for repository version {0}".format(
self.repository_version))
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(
url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos()
except Exception, err:
Logger.logger.exception(
"Cannot distribute repositories. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'ambari_repositories': installed_repositories,
'installed_repository_version': self.repository_version,
'stack_id': stack_id,
'package_installation_result': 'FAIL'
}
self.put_structured_out(self.structured_output)
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output[
'actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
else:
self.repo_version_with_build_number = None
# Initial list of versions, used to compute the new version installed
self.old_versions = get_hdp_versions()
try:
# It's possible for the process to receive a SIGTERM while installing the packages
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
else:
num_errors += 1
except Exception, err:
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))