def list_ambari_managed_repos(stack_name):
"""
Lists all repositories that are present at host
"""
stack_name = stack_name.upper()
# TODO : get it dynamically from the server
repository_names = [stack_name, stack_name + "-UTILS" ]
if OSCheck.is_ubuntu_family():
repo_dir = '/etc/apt/sources.list.d/'
elif OSCheck.is_redhat_family(): # Centos/RHEL 5/6
repo_dir = '/etc/yum.repos.d/'
elif OSCheck.is_suse_family():
repo_dir = '/etc/zypp/repos.d/'
else:
raise Fail('Can not dermine repo dir')
repos = []
for name in repository_names:
# List all files that match pattern
files = glob.glob(os.path.join(repo_dir, name) + '*')
for f in files:
filename = os.path.basename(f)
# leave out extension
reponame = os.path.splitext(filename)[0]
repos.append(reponame)
# get uniq strings
seen = set()
uniq = [s for s in repos if not (s in seen or seen.add(s))]
return uniq
def actionexecute(self, env):
config = Script.get_config()
version = default('/commandParams/version', None)
stack_name = default('/hostLevelParams/stack_name', "")
if not version:
raise Fail("Value is required for '/commandParams/version'")
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
min_ver = format_stack_version("2.2")
real_ver = format_stack_version(version)
if stack_name == "HDP":
if compare_versions(real_ver, min_ver) >= 0:
stack_selector_path = stack_tools.get_stack_tool_path(
stack_tools.STACK_SELECTOR_NAME)
cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all',
version)
code, out = shell.call(cmd, sudo=True)
if compare_versions(real_ver, format_stack_version("2.3")) >= 0:
# backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component]
for k, v in conf_select.get_package_dirs().iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'],
dir_def['current_dir'])
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info_json = config['hostLevelParams']['repo_info']
repo_info_dict = json.loads(repo_info_json)
for item in repo_info_dict["repositories"]:
base_url = item["base_url"]
repo_name = item["repo_name"]
repo_id = item["repo_id"]
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [repo_name] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
Repository(repo_id,
action = "create",
base_url = base_url,
mirror_list = None,
repo_file_name = repo_name,
repo_template = template,
components = ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")}
except Exception, exception:
Logger.logger.exception("ERROR: There was an unexpected error while updating repositories")
raise Fail("Failed to update repo files!")
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info = config['repositoryFile']
for item in repo_info["repositories"]:
base_url = item["baseUrl"]
repo_name = item["repoName"]
repo_id = item["repoId"]
distribution = item["distribution"] if "distribution" in item else None
components = item["components"] if "components" in item else None
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [distribution if distribution else repo_name] + \
[components.replace(",", " ") if components else self.UBUNTU_REPO_COMPONENTS_POSTFIX]
Repository(repo_id,
action = "create",
base_url = base_url,
mirror_list = None,
repo_file_name = repo_name,
repo_template = template,
components = ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")}
except Exception, exception:
Logger.logger.exception("ERROR: There was an unexpected error while updating repositories")
raise Fail("Failed to update repo files!")
def install_repos():
import params
template = "repo_suse_rhel.j2" if OSCheck.is_suse_family(
) or OSCheck.is_redhat_family() else "repo_ubuntu.j2"
_alter_repo("create", params.repo_info, template)
if params.service_repo_info:
_alter_repo("create", params.service_repo_info, template)
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
try:
command_repository = CommandRepository(config['repositoryFile'])
except KeyError:
raise Fail(
"The command repository indicated by 'repositoryFile' was not found"
)
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
self.repository_version = command_repository.version_string
# Select dict that contains parameters
try:
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
pass
self.stack_name = Script.get_stack_name()
if self.stack_name is None:
raise Fail("Cannot determine the stack name")
self.stack_root_folder = Script.get_stack_root()
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory")
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
try:
if 0 == len(command_repository.repositories):
Logger.warning(
"Repository list is empty. Ambari may not be managing the repositories for {0}."
.format(self.repository_version))
else:
Logger.info(
"Will install packages for repository version {0}".format(
self.repository_version))
create_repo_files(template, command_repository)
except Exception, err:
Logger.logger.exception(
"Cannot install repository files. Error: {0}".format(str(err)))
num_errors += 1
def actionexecute(self, env):
version = default('/commandParams/version', None)
if not version:
raise Fail("Value is required for '/commandParams/version'")
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
formatted_version = format_stack_version(version)
if not formatted_version:
raise Fail("Unable to determine a properly formatted stack version from {0}".format(version))
stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
# this script runs on all hosts; if this host doesn't have stack components,
# then don't invoke the stack tool
# (no need to log that it's skipped - the function will do that)
if is_host_skippable(stack_selector_path, formatted_version):
return
# invoke "set all"
cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version)
code, out = shell.call(cmd, sudo=True)
if code != 0:
raise Exception("Command '{0}' exit code is nonzero".format(cmd))
if check_stack_feature(StackFeature.CONFIG_VERSIONING, formatted_version):
# backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component]
for k, v in conf_select.get_package_dirs().iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'], dir_def['current_dir'])
def actionexecute(self, env):
config = Script.get_config()
version = default('/commandParams/version', None)
stack_name = default('/hostLevelParams/stack_name', "")
if not version:
raise Fail("Value is required for '/commandParams/version'")
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
real_ver = format_stack_version(version)
if real_ver and check_stack_feature(StackFeature.ROLLING_UPGRADE, real_ver):
stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME)
cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version)
code, out = shell.call(cmd, sudo=True)
if code != 0:
raise Exception("Command '{0}' exit code is nonzero".format(cmd))
if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver):
# backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component]
for k, v in conf_select.get_package_dirs().iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'], dir_def['current_dir'])
def get_lzo_packages(stack_version_unformatted):
lzo_packages = []
script_instance = Script.get_instance()
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo", "hadoop-lzo-native"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
if stack_version_unformatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
if OSCheck.is_ubuntu_family():
lzo_packages += [
script_instance.format_package_name(
"hadooplzo-${stack_version}"),
script_instance.format_package_name(
"hadooplzo-${stack_version}-native")
]
else:
lzo_packages += [
script_instance.format_package_name(
"hadooplzo_${stack_version}"),
script_instance.format_package_name(
"hadooplzo_${stack_version}-native")
]
else:
lzo_packages += ["hadoop-lzo"]
return lzo_packages
def actionexecute(self, env):
config = Script.get_config()
version = default('/commandParams/version', None)
stack_name = default('/hostLevelParams/stack_name', "")
if not version:
raise Fail("Value is required for '/commandParams/version'")
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
min_ver = format_hdp_stack_version("2.2")
real_ver = format_hdp_stack_version(version)
if stack_name == "HDP":
if compare_versions(real_ver, min_ver) >= 0:
cmd = ('hdp-select', 'set', 'all', version)
code, out = shell.call(cmd, sudo=True)
if compare_versions(real_ver, format_hdp_stack_version("2.3")) >= 0:
# backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component]
for k, v in conf_select.PACKAGE_DIRS.iteritems():
for dir_def in v:
link_config(dir_def['conf_dir'], dir_def['current_dir'])
def list_ambari_managed_repos(stack_name):
"""
Lists all repositories that are present at host
"""
stack_name = stack_name.upper()
# TODO : get it dynamically from the server
repository_names = [stack_name, stack_name + "-UTILS"]
if OSCheck.is_ubuntu_family():
repo_dir = '/etc/apt/sources.list.d/'
elif OSCheck.is_redhat_family(): # Centos/RHEL 5/6
repo_dir = '/etc/yum.repos.d/'
elif OSCheck.is_suse_family():
repo_dir = '/etc/zypp/repos.d/'
else:
raise Fail('Can not dermine repo dir')
repos = []
for name in repository_names:
# List all files that match pattern
files = glob.glob(os.path.join(repo_dir, name) + '*')
for f in files:
filename = os.path.basename(f)
# leave out extension
reponame = os.path.splitext(filename)[0]
repos.append(reponame)
# get uniq strings
seen = set()
uniq = [s for s in repos if not (s in seen or seen.add(s))]
return uniq
def get_lzo_packages():
lzo_packages = []
script_instance = Script.get_instance()
if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
lzo_packages += ["liblzo2-2"]
elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
stack_version_unformatted = stack_features.get_stack_feature_version(
Script.get_config()
) # only used to check stack_feature, NOT as package version!
if stack_version_unformatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
if OSCheck.is_ubuntu_family():
lzo_packages += [
script_instance.format_package_name(
"hadooplzo-${stack_version}"),
script_instance.format_package_name(
"hadooplzo-${stack_version}-native")
]
else:
lzo_packages += [
script_instance.format_package_name(
"hadooplzo_${stack_version}"),
script_instance.format_package_name(
"hadooplzo_${stack_version}-native")
]
return lzo_packages
def execute_transparent_huge_page_check(self, config):
Logger.info("Transparent huge page check started.")
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
transparent_huge_page_check_structured_output = {
"exit_code": 0,
"message":
str(re.search(thp_regex, file_content).groups()[0])
}
else:
transparent_huge_page_check_structured_output = {
"exit_code": 0,
"message": ""
}
Logger.info("Transparent huge page check completed.")
return transparent_huge_page_check_structured_output
def install_repository(self, url_info, repository_version, append_to_file):
template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else "repo_ubuntu.j2"
repo = {
'repoName': "{0}-{1}".format(url_info['name'], repository_version)
}
if not 'baseUrl' in url_info:
repo['baseurl'] = None
else:
repo['baseurl'] = url_info['baseUrl']
if not 'mirrorsList' in url_info:
repo['mirrorsList'] = None
else:
repo['mirrorsList'] = url_info['mirrorsList']
ubuntu_components = [url_info['name']
] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
file_name = self.REPO_FILE_NAME_PREFIX + repository_version
Repository(
repo['repoName'],
action="create",
base_url=repo['baseurl'],
mirror_list=repo['mirrorsList'],
repo_file_name=file_name,
repo_template=template,
append_to_file=append_to_file,
components=ubuntu_components, # ubuntu specific
)
return repo['repoName'], file_name
def install_repository(self, url_info, repository_version, append_to_file):
template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else "repo_ubuntu.j2"
repo = {
'repoName': "{0}-{1}".format(url_info['name'], repository_version)
}
if not 'baseUrl' in url_info:
repo['baseurl'] = None
else:
repo['baseurl'] = url_info['baseUrl']
if not 'mirrorsList' in url_info:
repo['mirrorsList'] = None
else:
repo['mirrorsList'] = url_info['mirrorsList']
ubuntu_components = [url_info['name']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
file_name = self.REPO_FILE_NAME_PREFIX + repository_version
Repository(repo['repoName'],
action = "create",
base_url = repo['baseurl'],
mirror_list = repo['mirrorsList'],
repo_file_name = file_name,
repo_template = template,
append_to_file = append_to_file,
components = ubuntu_components, # ubuntu specific
)
return repo['repoName'], file_name
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
# Install packages
packages_were_checked = False
try:
Package(self.get_base_packages_to_install())
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [
package[0] for package in packages_installed_before
]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'],
self.repository_version)
Package(
name,
use_repos=list(self.current_repo_files) if
OSCheck.is_ubuntu_family() else self.current_repositories,
skip_repos=[self.REPO_FILE_NAME_PREFIX +
"*"] if OSCheck.is_redhat_family() else [])
except Exception, err:
ret_code = 1
Logger.logger.exception(
"Package Manager failed to install packages. Error: {0}".
format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [
package[0] for package in packages_installed_after
]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [
package for package in packages_installed_after
if package not in packages_installed_before
]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace(
'.', '-')
else:
package_version_string = self.repository_version.replace(
'-', '_')
package_version_string = package_version_string.replace(
'.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string
in package):
Package(package, action="remove")
def install_repos():
import params
if params.host_sys_prepped:
return
template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu
_alter_repo("create", params.repo_info, template)
if params.service_repo_info:
_alter_repo("create", params.service_repo_info, template)
def install_repos():
import params
if params.host_sys_prepped:
return
template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu
_alter_repo("create", params.repo_info, template)
if params.service_repo_info:
_alter_repo("create", params.service_repo_info, template)
def getRepoDir(self):
if OSCheck.is_redhat_family():
return "/etc/yum.repos.d"
elif OSCheck.is_suse_family():
return "/etc/zypp/repos.d"
elif OSCheck.is_ubuntu_family():
return "/etc/apt/sources.list.d"
else:
raise Exception("Unsupported OS family '{0}'".format(
OSCheck.get_os_family()))
def get_lzo_packages(stack_version_unformatted):
lzo_packages = ["hadoop-lzo"]
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo", "hadoop-lzo-native"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
underscored_version = stack_version_unformatted.replace('.', '_')
dashed_version = stack_version_unformatted.replace('.', '-')
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
if hdp_stack_version != "" and compare_versions(hdp_stack_version,
'2.2') >= 0:
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += [format("hadooplzo_{underscored_version}_*")]
elif OSCheck.is_ubuntu_family():
lzo_packages += [format("hadooplzo_{dashed_version}_*")]
return lzo_packages
def install_repos():
import params_repo
#if OSCheck.is_ubuntu_family():
#cassandra_repo_file = params_repo.repo_ubuntu_path
#with open(path, "a") as f:
#f.write(params_repo.repo_ubuntu_content)
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
with open(params_repo.repo_rhel_suse_path, "w") as f:
f.writelines(params_repo.repo_rhel_suse_content)
else:
raise Exception('Cassandra repo file path not set for current OS.')
def get_clearcache_cmd(self):
if OSCheck.is_redhat_family():
Logger.info("Clear repository cache for the RedHat OS family");
return ("/usr/bin/yum", "clean", "all")
elif OSCheck.is_suse_family():
Logger.info("Clear repository cache for the SUSE OS family");
return ('/usr/bin/zypper', 'refresh')
elif OSCheck.is_ubuntu_family():
Logger.info("Clear repository cache for the Ubuntu OS family");
return ('/usr/bin/apt-get', 'update')
else:
raise Exception("Unsupported OS family: '{0}' ".format(OSCheck.get_os_family()))
def __init__(self, config, tags_to_skip):
self.tags_to_skip = tags_to_skip
# repo templates
repo_file = config['repositoryFile']
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
if is_empty(repo_file):
return
self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
self.command_repository = CommandRepository(repo_file)
def _clear_package_manager_cache(self):
package_manager_cmd = ""
if OSCheck.is_redhat_family():
package_manager_cmd = ("/usr/bin/yum", "clean", "metadata")
if OSCheck.is_suse_family():
package_manager_cmd = ("/usr/bin/zypper", "-q", "-n", "clean")
if OSCheck.is_ubuntu_family():
return
Logger.debug("Clearing repo manager metadata")
Execute(package_manager_cmd, logoutput=False, sudo=True)
def install_repos():
import params
if params.host_sys_prepped:
return
template = params.repo_rhel_suse if OSCheck.is_suse_family(
) or OSCheck.is_redhat_family() else params.repo_ubuntu
# use this newer way of specifying repositories, if available
if params.repo_file is not None:
create_repo_files(template, CommandRepository(params.repo_file))
return
_alter_repo("create", params.repo_info, template)
if params.service_repo_info:
_alter_repo("create", params.service_repo_info, template)
def _clear_package_manager_cache(self):
package_manager_cmd = ""
if OSCheck.is_redhat_family():
package_manager_cmd = ("/usr/bin/yum", "clean", "metadata")
if OSCheck.is_suse_family():
package_manager_cmd = ("/usr/bin/zypper", "-q", "-n", "clean")
if OSCheck.is_ubuntu_family():
return
Logger.debug("Clearing repo manager metadata")
Execute(package_manager_cmd, logoutput=False, sudo=True)
def get_elastic_config_path(default="/etc/default/elasticsearch"):
"""
Defines the path to the Elasticsearch environment file. This path will
differ based on the OS family.
:param default: The path used if the OS family is not recognized.
"""
path = default
if OSCheck.is_redhat_family():
path = "/etc/sysconfig/elasticsearch"
elif OSCheck.is_ubuntu_family():
path = "/etc/default/elasticsearch"
else:
Logger.error("Unexpected OS family; using default path={0}".format(path))
return path
def get_elastic_config_path(default="/etc/default/elasticsearch"):
"""
Defines the path to the Elasticsearch environment file. This path will
differ based on the OS family.
:param default: The path used if the OS family is not recognized.
"""
path = default
if OSCheck.is_redhat_family():
path = "/etc/sysconfig/elasticsearch"
elif OSCheck.is_ubuntu_family():
path = "/etc/default/elasticsearch"
else:
Logger.error("Unexpected OS family; using default path={0}".format(path))
return path
def get_lzo_packages(stack_version_unformatted):
lzo_packages = []
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo", "hadoop-lzo-native"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
if stack_version_unformatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
lzo_packages += ["hadooplzo_*"]
else:
lzo_packages += ["hadoop-lzo"]
return lzo_packages
def install_packages(self, package_list):
"""
Actually install the packages using the package manager.
:param package_list: List of package names to install
:return: Returns 0 if no errors were found, and 1 otherwise.
"""
ret_code = 0
# Clear cache of package manager right before installation of the packages
self._clear_package_manager_cache()
# Install packages
packages_were_checked = False
try:
Package(self.get_base_packages_to_install())
packages_installed_before = []
allInstalledPackages(packages_installed_before)
packages_installed_before = [package[0] for package in packages_installed_before]
packages_were_checked = True
filtered_package_list = self.filter_package_list(package_list)
for package in filtered_package_list:
name = self.format_package_name(package['name'], self.repository_version)
Package(name,
use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories,
skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else [])
except Exception, err:
ret_code = 1
Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err)))
# Remove already installed packages in case of fail
if packages_were_checked and packages_installed_before:
packages_installed_after = []
allInstalledPackages(packages_installed_after)
packages_installed_after = [package[0] for package in packages_installed_after]
packages_installed_before = set(packages_installed_before)
new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before]
if OSCheck.is_ubuntu_family():
package_version_string = self.repository_version.replace('.', '-')
else:
package_version_string = self.repository_version.replace('-', '_')
package_version_string = package_version_string.replace('.', '_')
for package in new_packages_installed:
if package_version_string and (package_version_string in package):
Package(package, action="remove")
def __init__(self, config):
"""
Constructor for RepositoryUtil
:type config dict
"""
# repo templates
repo_file = config['repositoryFile']
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
if is_empty(repo_file):
return
self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
self.command_repository = CommandRepository(repo_file)
def get_lzo_packages(stack_version_unformatted):
lzo_packages = []
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo", "hadoop-lzo-native"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
underscored_version = stack_version_unformatted.replace('.', '_')
dashed_version = stack_version_unformatted.replace('.', '-')
stack_version_formatted = format_stack_version(stack_version_unformatted)
if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0:
lzo_packages += ["hadooplzo_*"]
else:
lzo_packages += ["hadoop-lzo"]
return lzo_packages
def actionexecute(self, env):
summary = upgrade_summary.get_upgrade_summary()
if summary is None:
Logger.warning("There is no upgrade in progress")
return
if summary.associated_version is None:
Logger.warning(
"There is no version associated with the upgrade in progress")
return
if summary.orchestration != "STANDARD":
Logger.warning(
"The 'stack-select set all' command can only be invoked during STANDARD upgrades"
)
return
if summary.direction.lower(
) != Direction.UPGRADE or summary.is_downgrade_allowed or summary.is_revert:
Logger.warning(
"The 'stack-select set all' command can only be invoked during an UPGRADE which cannot be downgraded"
)
return
# other os?
if OSCheck.is_redhat_family():
cmd = ('/usr/bin/yum', 'clean', 'all')
code, out = shell.call(cmd, sudo=True)
stack_selector_path = stack_tools.get_stack_tool_path(
stack_tools.STACK_SELECTOR_NAME)
# this script runs on all hosts; if this host doesn't have stack components,
# then don't invoke the stack tool
# (no need to log that it's skipped - the function will do that)
if is_host_skippable(stack_selector_path, summary.associated_version):
return
# invoke "set all"
cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all',
summary.associated_version)
code, out = shell.call(cmd, sudo=True)
if code != 0:
raise Exception("Command '{0}' exit code is nonzero".format(cmd))
def actionexecute(self, env):
config = Script.get_config()
structured_output = {}
try:
repo_info_json = config['hostLevelParams']['repo_info']
repo_info_dict = json.loads(repo_info_json)
for item in repo_info_dict["repositories"]:
base_url = item["base_url"]
repo_name = item["repo_name"]
repo_id = item["repo_id"]
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_suse_family(
) or OSCheck.is_redhat_family() else repo_ubuntu
ubuntu_components = [repo_name
] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
Repository(
repo_id,
action="create",
base_url=base_url,
mirror_list=None,
repo_file_name=repo_name,
repo_template=template,
components=ubuntu_components, # ubuntu specific
)
structured_output["repo_update"] = {
"exit_code": 0,
"message": format("Repository files successfully updated!")
}
except Exception, exception:
Logger.logger.exception(
"ERROR: There was an unexpected error while updating repositories"
)
raise Fail("Failed to update repo files!")
def get_lzo_packages(stack_version_unformatted):
lzo_packages = []
script_instance = Script.get_instance()
if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo", "hadoop-lzo-native"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
if OSCheck.is_ubuntu_family():
lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") ,
script_instance.format_package_name("hadooplzo-${stack_version}-native")]
else:
lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
script_instance.format_package_name("hadooplzo_${stack_version}-native")]
else:
lzo_packages += ["hadoop-lzo"]
return lzo_packages
def get_lzo_packages():
lzo_packages = []
script_instance = Script.get_instance()
if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
lzo_packages += ["liblzo2-2"]
elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
lzo_packages += ["lzo"]
elif OSCheck.is_ubuntu_family():
lzo_packages += ["liblzo2-2"]
if OSCheck.is_ubuntu_family():
lzo_packages += [
script_instance.format_package_name("hadooplzo-${stack_version}"),
script_instance.format_package_name(
"hadooplzo-${stack_version}-native")
]
else:
lzo_packages += [
script_instance.format_package_name("hadooplzo_${stack_version}"),
script_instance.format_package_name(
"hadooplzo_${stack_version}-native")
]
return lzo_packages
master_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_master_heapsize'])
regionserver_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_regionserver_heapsize'])
regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
regionserver_xmn_percent = expect("/configurations/hbase-env/hbase_regionserver_xmn_ratio", float)
regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
hbase_regionserver_shutdown_timeout = expect('/configurations/hbase-env/hbase_regionserver_shutdown_timeout', int, 30)
phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)
has_phoenix = len(phoenix_hosts) > 0
underscored_version = stack_version_unformatted.replace('.', '_')
dashed_version = stack_version_unformatted.replace('.', '-')
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
phoenix_package = format("phoenix_{underscored_version}_*")
elif OSCheck.is_ubuntu_family():
phoenix_package = format("phoenix-{dashed_version}-.*")
pid_dir = status_params.pid_dir
tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
local_dir = config['configurations']['hbase-site']['hbase.local.dir']
ioengine_param = default('/configurations/hbase-site/hbase.bucketcache.ioengine', None)
client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf")
ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
# Select dict that contains parameters
try:
self.repository_version = config['roleParams']['repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
# Last try
self.repository_version = config['commandParams']['repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
stack_name = None
self.stack_root_folder = None
if stack_id and "-" in stack_id:
stack_split = stack_id.split("-")
if len(stack_split) == 2:
stack_name = stack_split[0].upper()
if stack_name in self.STACK_TO_ROOT_FOLDER:
self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[stack_name]
if self.stack_root_folder is None:
raise Fail("Cannot determine the stack's root directory by parsing the stack_id property, {0}".format(str(stack_id)))
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(self.repository_version))
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos()
except Exception, err:
Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err)))
num_errors += 1
def actionexecute(self, env):
num_errors = 0
# Parse parameters
config = Script.get_config()
repo_rhel_suse = config['configurations']['cluster-env'][
'repo_suse_rhel_template']
repo_ubuntu = config['configurations']['cluster-env'][
'repo_ubuntu_template']
template = repo_rhel_suse if OSCheck.is_redhat_family(
) or OSCheck.is_suse_family() else repo_ubuntu
# Handle a SIGTERM and SIGINT gracefully
signal.signal(signal.SIGTERM, self.abort_handler)
signal.signal(signal.SIGINT, self.abort_handler)
# Select dict that contains parameters
try:
self.repository_version = config['roleParams'][
'repository_version']
base_urls = json.loads(config['roleParams']['base_urls'])
package_list = json.loads(config['roleParams']['package_list'])
stack_id = config['roleParams']['stack_id']
except KeyError:
# Last try
self.repository_version = config['commandParams'][
'repository_version']
base_urls = json.loads(config['commandParams']['base_urls'])
package_list = json.loads(config['commandParams']['package_list'])
stack_id = config['commandParams']['stack_id']
# current stack information
self.current_hdp_stack_version = None
if 'stack_version' in config['hostLevelParams']:
current_stack_version_unformatted = str(
config['hostLevelParams']['stack_version'])
self.current_hdp_stack_version = format_hdp_stack_version(
current_stack_version_unformatted)
stack_name = None
self.stack_root_folder = None
if stack_id and "-" in stack_id:
stack_split = stack_id.split("-")
if len(stack_split) == 2:
stack_name = stack_split[0].upper()
if stack_name in self.STACK_TO_ROOT_FOLDER:
self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[
stack_name]
if self.stack_root_folder is None:
raise Fail(
"Cannot determine the stack's root directory by parsing the stack_id property, {0}"
.format(str(stack_id)))
if self.repository_version is None:
raise Fail("Cannot determine the repository version to install")
self.repository_version = self.repository_version.strip()
# Install/update repositories
installed_repositories = []
self.current_repositories = []
self.current_repo_files = set()
# Enable base system repositories
# We don't need that for RHEL family, because we leave all repos enabled
# except disabled HDP* ones
if OSCheck.is_suse_family():
self.current_repositories.append('base')
elif OSCheck.is_ubuntu_family():
self.current_repo_files.add('base')
Logger.info("Will install packages for repository version {0}".format(
self.repository_version))
try:
append_to_file = False
for url_info in base_urls:
repo_name, repo_file = self.install_repository(
url_info, append_to_file, template)
self.current_repositories.append(repo_name)
self.current_repo_files.add(repo_file)
append_to_file = True
installed_repositories = list_ambari_managed_repos()
except Exception, err:
Logger.logger.exception(
"Cannot distribute repositories. Error: {0}".format(str(err)))
num_errors += 1
installed_packages, repos = self.execute_existing_repos_and_installed_packages_check(config)
structured_output[CHECK_INSTALLED_PACKAGES] = installed_packages
structured_output[CHECK_EXISTING_REPOS] = repos
except Exception, exception :
print "There was an unknown error while checking installed packages and existing repositories: " + str(exception)
structured_output[CHECK_INSTALLED_PACKAGES] = {"exit_code" : 1, "message": str(exception)}
structured_output[CHECK_EXISTING_REPOS] = {"exit_code" : 1, "message": str(exception)}
# Here we are checking transparent huge page if CHECK_TRANSPARENT_HUGE_PAGE is in check_execute_list
if CHECK_TRANSPARENT_HUGE_PAGE in check_execute_list:
try :
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 0, "message": str(re.search(thp_regex,
file_content).groups()[0])}
else:
structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 0, "message": ""}
except Exception, exception :
print "There was an unknown error while getting transparent huge page data: " + str(exception)
structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 1, "message": str(exception)}
# this is necessary for HostCleanup to know later what were the results.
self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output)
regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)
has_phoenix = len(phoenix_hosts) > 0
if not has_phoenix and not phoenix_enabled:
exclude_packages = ['phoenix*']
else:
exclude_packages = []
underscored_version = stack_version_unformatted.replace('.', '_')
dashed_version = stack_version_unformatted.replace('.', '-')
if OSCheck.is_redhat_family() or OSCheck.is_suse_family():
phoenix_package = format("phoenix_{underscored_version}_*")
elif OSCheck.is_ubuntu_family():
phoenix_package = format("phoenix-{dashed_version}-.*")
pid_dir = status_params.pid_dir
tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
local_dir = config['configurations']['hbase-site']['hbase.local.dir']
client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf")
master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf")
regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf")
queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf")
ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]