def check_partial_install(self):
"""
If an installation did not complete successfully, check if installation was partially complete and
log the partially completed version to REPO_VERSION_HISTORY_FILE.
:return:
"""
Logger.info(
"Installation of packages failed. Checking if installation was partially complete"
)
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_stack_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
# Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
partial_install_version = next(iter(deltas)).strip()
write_actual_version_to_history_file(normalized_repo_version,
partial_install_version)
Logger.info("Version {0} was partially installed. ".format(
partial_install_version))
def get_lower_versions(self, current_version):
versions = get_stack_versions(stack_root)
Logger.info("available versions: {0}".format(str(versions)))
lover_versions = []
for version in versions:
if self.compare(version, current_version) < 0:
lover_versions.append(version)
Logger.info("version %s added to remove" % (version))
return lover_versions
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_stack_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if not command_repository.items:
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
new_repo_files = Script.repository_util.create_repo_files()
self.repo_files.update(new_repo_files)
except Exception as err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'package_installation_result': 'FAIL',
'repository_version_id': command_repository.version_id
}
self.put_structured_out(self.structured_output)
try:
# check package manager non-completed transactions
if self.repo_mgr.check_uncompleted_transactions():
self.repo_mgr.print_uncompleted_transaction_hint()
num_errors += 1
except Exception as e: # we need to ignore any exception
Logger.warning(
"Failed to check for uncompleted package manager transactions: "
+ str(e))
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_stack_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception as err:
num_errors += 1
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))
# Provide correct exit code
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
self._fix_default_links_for_current()
# if installing a version of HDP that needs some symlink love, then create them
if is_package_install_successful and 'actual_version' in self.structured_output:
self._relink_configurations_with_conf_select(
stack_id, self.structured_output['actual_version'])
class InstallPackages(Script):
"""
This script is a part of Rolling Upgrade workflow and is described at
appropriate design doc.
It installs repositories to the node and then installs packages.
For now, repositories are installed into individual files.
"""
UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if 0 == len(command_repository.repositories):
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
create_repo_files(template, command_repository)
except Exception, err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'package_installation_result': 'FAIL',
'repository_version_id': command_repository.version_id
}
self.put_structured_out(self.structured_output)
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_stack_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception, err:
num_errors += 1
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))
class InstallPackages(Script):
"""
This script is a part of Rolling Upgrade workflow and is described at
appropriate design doc.
It installs repositories to the node and then installs packages.
For now, repositories are installed into individual files.
"""
UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version_id = None
# Select dict that contains parameters
try:
self.repository_version = config['roleParams']['repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
if 'repository_version_id' in config['roleParams']:
self.repository_version_id = config['roleParams']['repository_version_id']
except KeyError:
# Last try
self.repository_version = config['commandParams']['repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
if 'repository_version_id' in config['commandParams']:
self.repository_version_id = config['commandParams']['repository_version_id']
# current stack information
self.current_stack_version_formatted = None
if 'stack_version' in config['hostLevelParams']:
current_stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted)
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(self.repository_version))
if 0 == len(base_urls):
Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos(self.stack_name)
except Exception, err:
Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'ambari_repositories': installed_repositories,
'installed_repository_version': self.repository_version,
'stack_id': stack_id,
'package_installation_result': 'FAIL'
}
if self.repository_version_id is not None:
self.structured_output['repository_version_id'] = self.repository_version_id
self.put_structured_out(self.structured_output)
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_stack_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output['package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception, err:
num_errors += 1
Logger.logger.exception("Could not install packages. Error: {0}".format(str(err)))
