def load_available_packages(self):
from ambari_commons.repo_manager import ManagerFactory
if self.available_packages_in_repos:
return self.available_packages_in_repos
config = self.get_config()
service_name = config['serviceName'] if 'serviceName' in config else None
repos = CommandRepository(config['repositoryFile'])
from resource_management.libraries.functions import lzo_utils
# remove repos with 'GPL' tag when GPL license is not approved
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
repos.items = [r for r in repos.items if not (repo_tags_to_skip & r.tags)]
repo_ids = [repo.repo_id for repo in repos.items]
Logger.info("Command repositories: {0}".format(", ".join(repo_ids)))
repos.items = [x for x in repos.items if (not x.applicable_services or service_name in x.applicable_services) ]
applicable_repo_ids = [repo.repo_id for repo in repos.items]
Logger.info("Applicable repositories: {0}".format(", ".join(applicable_repo_ids)))
pkg_provider = ManagerFactory.get()
try:
self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(repos)
except Exception as err:
Logger.exception("Unable to load available packages")
self.available_packages_in_repos = []
return self.available_packages_in_repos
def load_available_packages(self):
from ambari_commons.repo_manager import ManagerFactory
if self.available_packages_in_repos:
return self.available_packages_in_repos
config = self.get_config()
service_name = config['serviceName'] if 'serviceName' in config else None
repos = CommandRepository(config['repositoryFile'])
repo_ids = [repo.repo_id for repo in repos.items]
Logger.info("Command repositories: {0}".format(", ".join(repo_ids)))
repos.items = [x for x in repos.items if (not x.applicable_services or service_name in x.applicable_services) ]
applicable_repo_ids = [repo.repo_id for repo in repos.items]
Logger.info("Applicable repositories: {0}".format(", ".join(applicable_repo_ids)))
pkg_provider = ManagerFactory.get()
try:
self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(repos)
except Exception as err:
Logger.exception("Unable to load available packages")
self.available_packages_in_repos = []
return self.available_packages_in_repos
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if 0 == len(command_repository.repositories):
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
create_repo_files(template, command_repository)
except Exception, err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
def get_package_from_available(self, name, available_packages_in_repos=None):
"""
This function matches package names with ${stack_version} placeholder to actual package names from
Ambari-managed repository.
Package names without ${stack_version} placeholder are returned as is.
"""
if STACK_VERSION_PLACEHOLDER not in name:
return name
if not available_packages_in_repos:
available_packages_in_repos = self.load_available_packages()
from resource_management.libraries.functions.default import default
package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
package_regex = name.replace(STACK_VERSION_PLACEHOLDER, '(\d|{0})+'.format(package_delimiter)) + "$"
repo = default('/repositoryFile', None)
name_with_version = None
if repo:
command_repo = CommandRepository(repo)
version_str = command_repo.version_string.replace('.', package_delimiter).replace("-", package_delimiter)
name_with_version = name.replace(STACK_VERSION_PLACEHOLDER, version_str)
for package in available_packages_in_repos:
if re.match(package_regex, package):
return package
if name_with_version:
raise Fail("No package found for {0}(expected name: {1})".format(name, name_with_version))
else:
raise Fail("Cannot match package for regexp name {0}. Available packages: {1}".format(name, self.available_packages_in_repos))
def install_repos():
import params
if params.host_sys_prepped:
return
template = params.repo_rhel_suse if OSCheck.is_suse_family(
) or OSCheck.is_redhat_family() else params.repo_ubuntu
# use this newer way of specifying repositories, if available
if params.repo_file is not None:
create_repo_files(template, CommandRepository(params.repo_file))
return
_alter_repo("create", params.repo_info, template)
if params.service_repo_info:
_alter_repo("create", params.service_repo_info, template)
def get_current_component_version():
"""
Returns best available version for the component at different stages (install, start, stop)
:return best matched version or None
:rtype str|None
"""
from resource_management.core.exceptions import Fail
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.stack_select import get_role_component_current_stack_version
from resource_management.libraries.functions.repository_util import CommandRepository
version = default("/commandParams/version", None)
if not version:
repository = CommandRepository(default("/repositoryFile", {}))
if not repository.resolved:
try:
version = get_role_component_current_stack_version()
except (Fail, TypeError):
pass
else:
version = repository.version_string
return version
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if not command_repository.items:
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
new_repo_files = Script.repository_util.create_repo_files()
self.repo_files.update(new_repo_files)
except Exception as err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
# Build structured output with initial values
self.structured_output = {
'package_installation_result': 'FAIL',
'repository_version_id': command_repository.version_id
}
self.put_structured_out(self.structured_output)
try:
# check package manager non-completed transactions
if self.repo_mgr.check_uncompleted_transactions():
self.repo_mgr.print_uncompleted_transaction_hint()
num_errors += 1
except Exception as e: # we need to ignore any exception
Logger.warning(
"Failed to check for uncompleted package manager transactions: "
+ str(e))
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
# Initial list of versions, used to compute the new version installed
self.old_versions = get_stack_versions(self.stack_root_folder)
try:
is_package_install_successful = False
ret_code = self.install_packages(package_list)
if ret_code == 0:
self.structured_output[
'package_installation_result'] = 'SUCCESS'
self.put_structured_out(self.structured_output)
is_package_install_successful = True
else:
num_errors += 1
except Exception as err:
num_errors += 1
Logger.logger.exception(
"Could not install packages. Error: {0}".format(str(err)))
# Provide correct exit code
if num_errors > 0:
raise Fail("Failed to distribute repositories/install packages")
self._fix_default_links_for_current()
# if installing a version of HDP that needs some symlink love, then create them
if is_package_install_successful and 'actual_version' in self.structured_output:
self._relink_configurations_with_conf_select(
stack_id, self.structured_output['actual_version'])
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
config = self.get_config()
agent_stack_retry_on_unavailability = cbool(
config['ambariLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = cint(
config['ambariLevelParams']['agent_stack_retry_count'])
# Install packages
packages_were_checked = False
packages_installed_before = []
stack_selector_package = stack_tools.get_stack_tool_package(
stack_tools.STACK_SELECTOR_NAME)
try:
# install the stack-selector; we need to supply the action as "upgrade" here since the normal
# install command will skip if the package is already installed in the system.
# This is required for non-versioned components, like stack-select, since each version of
# the stack comes with one. Also, scope the install by repository since we need to pick a
# specific repo that the stack-select tools are coming out of in case there are multiple
# patches installed
repositories = config['repositoryFile']['repositories']
command_repos = CommandRepository(config['repositoryFile'])
repository_ids = [
repository['repoId'] for repository in repositories
]
repos_to_use = {}
for repo_id in repository_ids:
if repo_id in self.repo_files:
repos_to_use[repo_id] = self.repo_files[repo_id]
self.repo_mgr.upgrade_package(
stack_selector_package,
RepoCallContext(ignore_errors=False,
use_repos=repos_to_use,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count))
packages_installed_before = self.repo_mgr.installed_packages()
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
try:
available_packages_in_repos = self.repo_mgr.get_available_packages_in_repos(
command_repos)
except Exception:
available_packages_in_repos = []
installation_context = RepoCallContext(
ignore_errors=False,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
for package in filtered_package_list:
name = self.get_package_from_available(
package['name'], available_packages_in_repos)
# This enables upgrading non-versioned packages, despite the fact they exist.
# Needed by 'mahout' which is non-version but have to be updated
self.repo_mgr.upgrade_package(name, installation_context)
except Exception as err:
ret_code = 1
Logger.logger.error(
"Package Manager failed to install packages: {0}".format(
str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = self.repo_mgr.installed_packages()
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace(
'.', '-')
else:
package_version_string = self.repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
self.repo_mgr.remove_package(package,
RepoCallContext())
if not self.repo_mgr.verify_dependencies():
ret_code = 1
Logger.logger.error("Failure while verifying dependencies")
Logger.logger.error(
"*******************************************************************************"
)
Logger.logger.error(
"Manually verify and fix package dependencies and then re-run install_packages"
)
Logger.logger.error(
"*******************************************************************************"
)
# Compute the actual version in order to save it in structured out
try:
if ret_code == 0:
self.compute_actual_version()
else:
self.check_partial_install()
except Fail as err:
ret_code = 1
Logger.logger.exception(
"Failure while computing actual version. Error: {0}".format(
str(err)))
return ret_code
def load_available_packages(self):
if self.available_packages_in_repos:
return self.available_packages_in_repos
pkg_provider = get_provider("Package")
try:
self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(CommandRepository(self.get_config()['repositoryFile']))
except Exception as err:
Logger.exception("Unable to load available packages")
self.available_packages_in_repos = []
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version_id = None
base_urls = []
# Select dict that contains parameters
try:
if 'base_urls' in config['roleParams']:
base_urls = json.loads(config['roleParams']['base_urls'])
self.repository_version = config['roleParams']['repository_version']
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
if 'repository_version_id' in config['roleParams']:
self.repository_version_id = config['roleParams']['repository_version_id']
except KeyError:
pass
# current stack information
self.current_stack_version_formatted = None
if 'stack_version' in config['hostLevelParams']:
current_stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted)
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(self.repository_version))
if 0 == len(base_urls):
Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version))
try:
if 'repositoryFile' in config:
create_repo_files(template, CommandRepository(config['repositoryFile']))
else:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
except Exception, err:
Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1