def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
# Install packages
packages_were_checked = False
try:
Package(
"hdp-select",
action="upgrade",
)
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'])
Package(
name,
action=
"upgrade" # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated
)
except Exception, err:
ret_code = 1
Logger.logger.exception(
"Package Manager failed to install packages. Error: {0}".
format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace(
'.', '-')
else:
package_version_string = self.repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
Package(package, action="remove")
def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock):
get_os_family_mock.return_value = 'suse'
stringToRead = """Refreshing service 'susecloud'.
Loading repository data...
Reading installed packages...
S | Name | Type | Version | Arch | Repository
--+-----------------------------------+---------+------------------------+--------+----------------------
i | ConsoleKit | package | 0.2.10-64.65.1 | x86_64 | SLES11-SP1-Updates
i | gweb | package | 2.2.0-99 | noarch | Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15
i | hadoop | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP
i | hadoop-libhdfs | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP
i | ambari-server | package | 1.2.4.9-1 | noarch | Ambari 1.x
i | hdp_mon_ganglia_addons | package | 1.2.4.9-1 | noarch | Ambari 1.x
i | Minimal | pattern | 11-38.13.9 | x86_64 | SLES11-SP1"""
result = {}
result['out'] = stringToRead
result['err'] = ""
result['retCode'] = 0
spwt_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(7, len(installedPackages))
self.assertTrue(installedPackages[1][0], "gweb")
self.assertTrue(installedPackages[3][2], "HDP")
self.assertTrue(installedPackages[6][1], "11-38.13.9")
def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock):
get_os_family_mock.return_value = 'suse'
stringToRead = """Refreshing service 'susecloud'.
Loading repository data...
Reading installed packages...
S | Name | Type | Version | Arch | Repository
--+-----------------------------------+---------+------------------------+--------+----------------------
i | ConsoleKit | package | 0.2.10-64.65.1 | x86_64 | SLES11-SP1-Updates
i | gweb | package | 2.2.0-99 | noarch | Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15
i | hadoop | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP
i | hadoop-libhdfs | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP
i | ambari-server | package | 1.2.4.9-1 | noarch | Ambari 1.x
i | hdp_mon_ganglia_addons | package | 1.2.4.9-1 | noarch | Ambari 1.x
i | Minimal | pattern | 11-38.13.9 | x86_64 | SLES11-SP1"""
result = {}
result['out'] = stringToRead
result['err'] = ""
result['retCode'] = 0
spwt_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(7, len(installedPackages))
self.assertTrue(installedPackages[1][0], "gweb")
self.assertTrue(installedPackages[3][2], "HDP")
self.assertTrue(installedPackages[6][1], "11-38.13.9")
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
# Install packages
packages_were_checked = False
try:
Package(self.get_base_packages_to_install())
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'],
self.repository_version)
Package(
name,
use_repos=list(self.current_repo_files) if
OSCheck.is_ubuntu_family() else self.current_repositories,
skip_repos=[self.REPO_FILE_NAME_PREFIX +
"*"] if OSCheck.is_redhat_family() else [])
except Exception, err:
ret_code = 1
Logger.logger.exception(
"Package Manager failed to install packages. Error: {0}".
format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace(
'.', '-')
else:
package_version_string = self.repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
Package(package, action="remove")
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
config = self.get_config()
agent_stack_retry_on_unavailability = cbool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = cint(config['hostLevelParams']['agent_stack_retry_count'])
# Install packages
packages_were_checked = False
stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
try:
Package(stack_selector_package,
action="upgrade",
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
)
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [package[0] for package in packages_installed_before]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'])
Package(name,
action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count
)
except Exception, err:
ret_code = 1
Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [package[0] for package in packages_installed_after]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace('.', '-')
else:
package_version_string = self.repository_version.replace('-', '_')
package_version_string = package_version_string.replace('.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string in package):
Package(package, action="remove")
def test_analyze_yum_output_err(self, subprocessWithTimeout_mock, get_os_family_mock):
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
result = {}
result['out'] = ""
result['err'] = ""
result['retCode'] = 1
subprocessWithTimeout_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(installedPackages, [])
def test_analyze_yum_output_err(self, subprocessWithTimeout_mock, get_os_family_mock):
get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
result = {}
result['out'] = ""
result['err'] = ""
result['retCode'] = 1
subprocessWithTimeout_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(installedPackages, [])
def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock):
get_os_family_mock.return_value = 'redhat'
stringToRead = """Loaded plugins: amazon-id, product-id, rhui-lb, security, subscription-manager
Updating certificate-based repositories.
Installed Packages
AMBARI.dev.noarch 1.x-1.el6 installed
PyXML.x86_64 0.8.4-19.el6 @koji-override-0
Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch
3-7.el6 @koji-override-0
hcatalog.noarch 0.11.0.1.3.0.0-107.el6
@HDP-1.3.0
hesiod.x86_64 3.1.0-19.el6 @koji-override-0/$releasever
hive.noarch 0.11.0.1.3.0.0-107.el6
@HDP-1.3.0
oracle-server-db.x86 1.3.17-2
@Oracle-11g
ambari-log4j.noarch 1.2.5.9-1 @AMBARI.dev-1.x
libconfuse.x86_64 2.7-4.el6 @HDP-epel"""
result = {}
result['out'] = stringToRead
result['err'] = ""
result['retCode'] = 0
subprocessWithTimeout_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(9, len(installedPackages))
for package in installedPackages:
self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86",
"Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
"hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch", "libconfuse.x86_64"])
self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"])
self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0",
"koji-override-0/$releasever", "AMBARI.dev-1.x", "Oracle-11g", "HDP-epel"])
packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"],
installedPackages)
self.assertEqual(4, len(packages))
expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
"hesiod.x86_64", "hive.noarch"]
for package in expected:
self.assertTrue(package in packages)
detailedPackages = packages_analyzer.getPackageDetails(installedPackages, packages)
self.assertEqual(4, len(detailedPackages))
for package in detailedPackages:
self.assertTrue(package['version'] in ["1.x-1.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6"])
self.assertTrue(package['repoName'] in ["installed", "koji-override-0", "HDP-1.3.0",
"koji-override-0/$releasever"])
self.assertFalse(package['repoName'] in ["AMBARI.dev-1.x"])
def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock):
get_os_family_mock.return_value = 'redhat'
stringToRead = """Loaded plugins: amazon-id, product-id, rhui-lb, security, subscription-manager
Updating certificate-based repositories.
Installed Packages
AMBARI.dev.noarch 1.x-1.el6 installed
PyXML.x86_64 0.8.4-19.el6 @koji-override-0
Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch
3-7.el6 @koji-override-0
hcatalog.noarch 0.11.0.1.3.0.0-107.el6
@HDP-1.3.0
hesiod.x86_64 3.1.0-19.el6 @koji-override-0/$releasever
hive.noarch 0.11.0.1.3.0.0-107.el6
@HDP-1.3.0
oracle-server-db.x86 1.3.17-2
@Oracle-11g
ambari-log4j.noarch 1.2.5.9-1 @AMBARI.dev-1.x
libconfuse.x86_64 2.7-4.el6 @HDP-epel"""
result = {}
result['out'] = stringToRead
result['err'] = ""
result['retCode'] = 0
subprocessWithTimeout_mock.return_value = result
installedPackages = []
packages_analyzer.allInstalledPackages(installedPackages)
self.assertEqual(9, len(installedPackages))
for package in installedPackages:
self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86",
"Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
"hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch", "libconfuse.x86_64"])
self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"])
self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0",
"koji-override-0/$releasever", "AMBARI.dev-1.x", "Oracle-11g", "HDP-epel"])
packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"],
installedPackages)
self.assertEqual(4, len(packages))
expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch",
"hesiod.x86_64", "hive.noarch"]
for package in expected:
self.assertTrue(package in packages)
detailedPackages = packages_analyzer.getPackageDetails(installedPackages, packages)
self.assertEqual(4, len(detailedPackages))
for package in detailedPackages:
self.assertTrue(package['version'] in ["1.x-1.el6", "3-7.el6", "3.1.0-19.el6",
"0.11.0.1.3.0.0-107.el6"])
self.assertTrue(package['repoName'] in ["installed", "koji-override-0", "HDP-1.3.0",
"koji-override-0/$releasever"])
self.assertFalse(package['repoName'] in ["AMBARI.dev-1.x"])
def get_packages_to_remove(self, version):
packages = []
formated_version = version.replace('.', '_').replace('-', '_')
all_installed_packages = []
allInstalledPackages(all_installed_packages)
all_installed_packages = [
package[0] for package in all_installed_packages
]
for package in all_installed_packages:
if formated_version in package and self.stack_tool_package not in package:
packages.append(package)
Logger.info("%s added to remove" % (package))
return packages
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
# Clear cache of package manager right before installation of the packages
self._clear_package_manager_cache()
# Install packages
packages_were_checked = False
try:
Package(self.get_base_packages_to_install())
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [package[0] for package in packages_installed_before]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'], self.repository_version)
Package(name,
use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories,
skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else [])
except Exception, err:
ret_code = 1
Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [package[0] for package in packages_installed_after]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace('.', '-')
else:
package_version_string = self.repository_version.replace('-', '_')
package_version_string = package_version_string.replace('.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string in package):
Package(package, action="remove")
def execute_existing_repos_and_installed_packages_check(self, config):
installedPackages = []
availablePackages = []
packages_analyzer.allInstalledPackages(installedPackages)
packages_analyzer.allAvailablePackages(availablePackages)
repos = []
packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
self.IGNORE_PACKAGES_FROM_REPOS, repos)
packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
self.ADDITIONAL_PACKAGES, installedPackages)
allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages)
repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS)
return installedPackages, repos
def execute_existing_repos_and_installed_packages_check(self, config):
installedPackages = []
availablePackages = []
packages_analyzer.allInstalledPackages(installedPackages)
packages_analyzer.allAvailablePackages(availablePackages)
repos = []
packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages,
self.IGNORE_PACKAGES_FROM_REPOS, repos)
packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages)
additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames(
self.ADDITIONAL_PACKAGES, installedPackages)
allPackages = list(set(packagesInstalled + additionalPkgsInstalled))
installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages)
repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS)
return installedPackages, repos
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
config = self.get_config()
agent_stack_retry_on_unavailability = cbool(
config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = cint(
config['hostLevelParams']['agent_stack_retry_count'])
# Install packages
packages_were_checked = False
stack_selector_package = stack_tools.get_stack_tool_package(
stack_tools.STACK_SELECTOR_NAME)
try:
Package(stack_selector_package,
action="upgrade",
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
try:
available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(
config['repositoryFile']['repositories'])
except Exception:
available_packages_in_repos = []
for package in filtered_package_list:
name = self.get_package_from_available(
package['name'], available_packages_in_repos)
Package(
name,
action=
"upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except Exception as err:
ret_code = 1
Logger.logger.exception(
"Package Manager failed to install packages. Error: {0}".
format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace(
'.', '-')
else:
package_version_string = self.repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
Package(package, action="remove")
if not verifyDependencies():
ret_code = 1
Logger.logger.error("Failure while verifying dependencies")
Logger.logger.error(
"*******************************************************************************"
)
Logger.logger.error(
"Manually verify and fix package dependencies and then re-run install_packages"
)
Logger.logger.error(
"*******************************************************************************"
)
# Compute the actual version in order to save it in structured out
try:
if ret_code == 0:
self.compute_actual_version()
else:
self.check_partial_install()
except Fail as err:
ret_code = 1
Logger.logger.exception(
"Failure while computing actual version. Error: {0}".format(
str(err)))
return ret_code
class InstallPackages(Script):
"""
This script is a part of Rolling Upgrade workflow and is described at
appropriate design doc.
It installs repositories to the node and then installs packages.
For now, repositories are installed into individual files.
"""
UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
REPO_FILE_NAME_PREFIX = 'HDP-'
def actionexecute(self, env):
delayed_fail = False
package_install_result = False
# Parse parameters
config = Script.get_config()
# Select dict that contains parameters
try:
repository_version = config['roleParams']['repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
# Last try
repository_version = config['commandParams']['repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
# Install/update repositories
installed_repositories = []
current_repositories = [
'base'
] # Some our packages are installed from the base repo
current_repo_files = set(['base'])
old_versions = self.hdp_versions()
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(
url_info, repository_version, append_to_file)
current_repositories.append(repo_name)
current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos()
except Exception, err:
print "Can not distribute repositories."
print traceback.format_exc()
delayed_fail = True
# Install packages
if not delayed_fail:
packages_were_checked = False
try:
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'],
repository_version)
Package(
name,
use_repos=list(current_repo_files) if
OSCheck.is_ubuntu_family() else current_repositories)
package_install_result = True
except Exception, err:
print "Can not install packages."
print traceback.format_exc()
delayed_fail = True
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = repository_version.replace(
'.', '-')
else:
package_version_string = repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
Package(package, action="remove")
