def test_is_redhat_family(self, get_os_family_mock):
get_os_family_mock.return_value = "redhat"
self.assertEqual(OSCheck.is_redhat_family(), True)
get_os_family_mock.return_value = "troll_os"
self.assertEqual(OSCheck.is_redhat_family(), False)
def get_ntp_service():
if OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
return ("chronyd", "ntpd",)
elif OSCheck.is_redhat_family():
return ("ntpd",)
elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
return ("ntp",)
def test_is_redhat_family(self, get_os_family_mock):
get_os_family_mock.return_value = "redhat"
self.assertEqual(OSCheck.is_redhat_family(), True)
get_os_family_mock.return_value = "troll_os"
self.assertEqual(OSCheck.is_redhat_family(), False)
def get_postgre_hba_dir(OS_FAMILY):
"""Return postgre hba dir location depends on OS.
Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3
1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3
2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
"""
if OSCheck.is_ubuntu_family():
# Like: /etc/postgresql/9.1/main/
return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(),
"main")
elif OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
return PG_HBA_ROOT_DEFAULT
else:
if not os.path.isfile(get_pg_hba_init_files()):
# Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
os.symlink(glob.glob(get_pg_hba_init_files() + '*')[0],
get_pg_hba_init_files())
# Get postgres_data location (default: /var/lib/pgsql/data)
cmd = "alias exit=return; source " + get_pg_hba_init_files() + " status &>/dev/null; echo $PGDATA"
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(PG_HBA_ROOT, err) = p.communicate()
if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:
return PG_HBA_ROOT.strip()
else:
return PG_HBA_ROOT_DEFAULT
def get_postgre_hba_dir(OS_FAMILY):
"""Return postgre hba dir location depends on OS.
Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3
1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3
2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
"""
if OSCheck.is_ubuntu_family():
# Like: /etc/postgresql/9.1/main/
return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(),
"main")
elif OSCheck.is_redhat_family() and int(
OSCheck.get_os_major_version()) >= 7:
return PG_HBA_ROOT_DEFAULT
else:
if not os.path.isfile(get_pg_hba_init_files()):
# Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
os.symlink(
glob.glob(get_pg_hba_init_files() + '*')[0],
get_pg_hba_init_files())
# Get postgres_data location (default: /var/lib/pgsql/data)
cmd = "alias exit=return; source " + get_pg_hba_init_files(
) + " status &>/dev/null; echo $PGDATA"
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(PG_HBA_ROOT, err) = p.communicate()
if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:
return PG_HBA_ROOT.strip()
else:
return PG_HBA_ROOT_DEFAULT
def find_repo_files_for_repos(self, repoNames):
repoFiles = []
osType = OSCheck.get_os_family()
repoNameList = []
for repoName in repoNames:
if len(repoName.strip()) > 0:
repoNameList.append("[" + repoName + "]")
repoNameList.append("name=" + repoName)
if repoNameList:
# get list of files
if OSCheck.is_suse_family():
fileList = self.get_files_in_dir(REPO_PATH_SUSE)
elif OSCheck.is_redhat_family():
fileList = self.get_files_in_dir(REPO_PATH_RHEL)
elif OSCheck.is_ubuntu_family():
fileList = self.get_files_in_dir(REPO_PATH_UBUNTU)
else:
logger.warn("Unsupported OS type, cannot get repository location.")
return []
if fileList:
for filePath in fileList:
with open(filePath, 'r') as file:
content = file.readline()
while (content != "" ):
for repoName in repoNameList:
if content.find(repoName) == 0 and filePath not in repoFiles:
repoFiles.append(filePath)
break;
content = file.readline()
return repoFiles
def verifyDependencies():
"""
Verify that we have no dependency issues in package manager. Dependency issues could appear because of aborted or terminated
package installation process or invalid packages state after manual modification of packages list on the host
:return True if no dependency issues found, False if dependency issue present
:rtype bool
"""
check_str = None
cmd = None
if OSCheck.is_redhat_family():
cmd = ['/usr/bin/yum', '-d', '0', '-e', '0', 'check', 'dependencies']
check_str = "has missing requires|Error:"
elif OSCheck.is_suse_family():
cmd = ['/usr/bin/zypper', '--quiet', '--non-interactive' 'verify', '--dry-run']
check_str = "\d+ new package(s)? to install"
elif OSCheck.is_ubuntu_family():
cmd = ['/usr/bin/apt-get', '-qq', 'check']
check_str = "has missing dependency|E:"
if check_str is None or cmd is None:
raise Fail("Unsupported OSFamily on the Agent Host")
code, out = rmf_shell.checked_call(cmd, sudo=True)
output_regex = re.compile(check_str)
if code or (out and output_regex.search(out)):
err_msg = Logger.filter_text("Failed to verify package dependencies. Execution of '%s' returned %s. %s" % (cmd, code, out))
Logger.error(err_msg)
return False
return True
def get_ntp_service():
if OSCheck.is_redhat_family():
return ("ntpd", "chronyd",)
elif OSCheck.is_suse_family():
return ("ntpd", "ntp",)
elif OSCheck.is_ubuntu_family():
return ("ntp", "chrony",)
def find_repo_files_for_repos(self, repoNames):
repoFiles = []
osType = OSCheck.get_os_family()
repoNameList = []
for repoName in repoNames:
if len(repoName.strip()) > 0:
repoNameList.append("[" + repoName + "]")
repoNameList.append("name=" + repoName)
if repoNameList:
# get list of files
if OSCheck.is_suse_family():
fileList = self.get_files_in_dir(REPO_PATH_SUSE)
elif OSCheck.is_redhat_family():
fileList = self.get_files_in_dir(REPO_PATH_RHEL)
elif OSCheck.is_ubuntu_family():
fileList = self.get_files_in_dir(REPO_PATH_UBUNTU)
else:
logger.warn(
"Unsupported OS type, cannot get repository location.")
return []
if fileList:
for filePath in fileList:
with open(filePath, 'r') as file:
content = file.readline()
while (content != ""):
for repoName in repoNameList:
if content.find(
repoName
) == 0 and filePath not in repoFiles:
repoFiles.append(filePath)
break
content = file.readline()
return repoFiles
def getRepoDir(self):
if OSCheck.is_redhat_family():
return "/etc/yum.repos.d"
elif OSCheck.is_suse_family():
return "/etc/zypp/repos.d"
elif OSCheck.is_ubuntu_family():
return "/etc/apt/sources.list.d"
else:
raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def get_erase_cmd():
if OSCheck.is_redhat_family():
return "yum erase -y {0}"
elif OSCheck.is_suse_family():
return "zypper -n -q remove {0}"
elif OSCheck.is_ubuntu_family():
return "/usr/bin/apt-get -y -q remove {0}"
else:
raise Exception("Unsupported OS family '{0}', cannot remove package. ".format(OSCheck.get_os_family()))
def get_pg_hba_init_files():
if OSCheck.is_ubuntu_family():
return '/etc/postgresql'
elif OSCheck.is_redhat_family():
return '/etc/rc.d/init.d/postgresql'
elif OSCheck.is_suse_family():
return '/etc/init.d/postgresql'
else:
raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def get_erase_cmd():
if OSCheck.is_redhat_family():
return "yum erase -y {0}"
elif OSCheck.is_suse_family():
return "zypper -n -q remove {0}"
elif OSCheck.is_ubuntu_family():
return "/usr/bin/apt-get -y -q remove {0}"
else:
raise Exception("Unsupported OS family '{0}', cannot remove package. ".format(OSCheck.get_os_family()))
def get_pg_hba_init_files():
if OSCheck.is_ubuntu_family():
return '/etc/postgresql'
elif OSCheck.is_redhat_family():
return '/etc/rc.d/init.d/postgresql'
elif OSCheck.is_suse_family():
return '/etc/init.d/postgresql'
else:
raise Exception("Unsupported OS family '{0}'".format(
OSCheck.get_os_family()))
def getFirewallObject(self):
if OSCheck.is_ubuntu_family():
return UbuntuFirewallChecks()
elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
return Fedora18FirewallChecks()
elif OSCheck.is_redhat_family() and int(self.OS_VERSION) >= 7:
return RedHat7FirewallChecks()
elif OSCheck.is_suse_family():
return SuseFirewallChecks()
else:
return FirewallChecks()
def getFirewallObject(self):
if OSCheck.is_ubuntu_family():
return UbuntuFirewallChecks()
elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
return Fedora18FirewallChecks()
elif OSCheck.is_redhat_family() and int(self.OS_VERSION) >= 7:
return RedHat7FirewallChecks()
elif OSCheck.is_suse_family():
return SuseFirewallChecks()
else:
return FirewallChecks()
def allAvailablePackages(allAvailablePackages):
osType = OSCheck.get_os_family()
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
["sudo", "zypper", "search", "--uninstalled-only", "--details"],
allAvailablePackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(["sudo", "yum", "list", "available"],
'Available Packages', allAvailablePackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(LIST_AVAILABLE_PACKAGES_UBUNTU,
allAvailablePackages)
def get_ambari_repo_file_full_name():
if OSCheck.is_ubuntu_family():
ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
elif OSCheck.is_redhat_family():
ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
elif OSCheck.is_suse_family():
ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
elif OSCheck.is_windows_family():
ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
else:
raise Exception('Ambari repo file path not set for current OS.')
return ambari_repo_file
def allAvailablePackages(allAvailablePackages):
if OSCheck.is_suse_family():
return _lookUpZypperPackages([
AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search",
"--uninstalled-only", "--details"
], allAvailablePackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
[AMBARI_SUDO_BINARY, "yum", "list", "available"],
'Available Packages', allAvailablePackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(
LIST_AVAILABLE_PACKAGES_UBUNTU % GREP_REPO_EXCLUDE_SYSTEM,
allAvailablePackages)
def get_ambari_repo_file_full_name():
if OSCheck.is_ubuntu_family():
ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
elif OSCheck.is_redhat_family():
ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
elif OSCheck.is_suse_family():
ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
elif OSCheck.is_windows_family():
ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
else:
raise Exception('Ambari repo file path not set for current OS.')
return ambari_repo_file
def getTransparentHugePage(self):
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = self.THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = self.THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
return re.search(thp_regex, file_content).groups()[0]
else:
return ""
def getTransparentHugePage(self):
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = self.THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = self.THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
return re.search(thp_regex, file_content).groups()[0]
else:
return ""
def getTransparentHugePage(self):
file_name = None
if OSCheck.is_ubuntu_family():
file_name = self.THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = self.THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
return HostInfoLinux.THP_REGEXP.search(
file_content).groups()[0]
else:
return ""
def allAvailablePackages(allAvailablePackages):
osType = OSCheck.get_os_family()
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
["sudo", "zypper", "search", "--uninstalled-only", "--details"],
allAvailablePackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
["sudo", "yum", "list", "available"],
'Available Packages',
allAvailablePackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(
LIST_AVAILABLE_PACKAGES_UBUNTU,
allAvailablePackages)
def allInstalledPackages(allInstalledPackages):
"""
All installed packages in system
"""
if OSCheck.is_suse_family():
return _lookUpZypperPackages([
AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search",
"--installed-only", "--details"
], allInstalledPackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
[AMBARI_SUDO_BINARY, "yum", "list", "installed"],
'Installed Packages', allInstalledPackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(LIST_INSTALLED_PACKAGES_UBUNTU,
allInstalledPackages)
def allInstalledPackages(allInstalledPackages):
"""
All installed packages in system
"""
osType = OSCheck.get_os_family()
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
["sudo", "zypper", "search", "--installed-only", "--details"],
allInstalledPackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(["sudo", "yum", "list", "installed"],
'Installed Packages', allInstalledPackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(LIST_INSTALLED_PACKAGES_UBUNTU,
allInstalledPackages)
def get_available_packages_in_repos(repositories):
"""
Gets all (both installed and available) packages that are available at given repositories.
:param repositories: from command configs like config['repositoryFile']['repositories']
:return: installed and available packages from these repositories
"""
available_packages = []
installed_packages = []
available_packages_in_repos = []
repo_ids = [repository['repoId'] for repository in repositories]
if OSCheck.is_ubuntu_family():
allInstalledPackages(installed_packages)
repo_urls = [repository['baseUrl'] for repository in repositories]
repo_urls = [repo_url.replace("http://", "") for repo_url in repo_urls]
repo_urls = [repo_url.replace("/", "_") for repo_url in repo_urls]
for url in repo_urls:
_lookUpAptPackages(LIST_AVAILABLE_PACKAGES_UBUNTU % url,
available_packages)
for package in installed_packages:
if url in package[2]:
available_packages_in_repos.append(package[0])
for package in available_packages:
available_packages_in_repos.append(package[0])
elif OSCheck.is_suse_family():
for repo in repo_ids:
_lookUpZypperPackages([
AMBARI_SUDO_BINARY, "zypper", "--no-gpg-checks", "search",
"--details", "--repo", repo
], available_packages)
available_packages_in_repos += [
package[0] for package in available_packages
]
elif OSCheck.is_redhat_family():
for repo in repo_ids:
_lookUpYumPackages([
AMBARI_SUDO_BINARY, "yum", "list", "available",
"--disablerepo=*", "--enablerepo=" + repo
], 'Available Packages', available_packages)
_lookUpYumPackages([
AMBARI_SUDO_BINARY, "yum", "list", "installed",
"--disablerepo=*", "--enablerepo=" + repo
], 'Installed Packages', installed_packages)
available_packages_in_repos += [
package[0] for package in available_packages + installed_packages
]
return available_packages_in_repos
def allInstalledPackages(allInstalledPackages):
"""
All installed packages in system
"""
osType = OSCheck.get_os_family()
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
["sudo", "zypper", "search", "--installed-only", "--details"],
allInstalledPackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
["sudo", "yum", "list", "installed"],
'Installed Packages',
allInstalledPackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(
LIST_INSTALLED_PACKAGES_UBUNTU,
allInstalledPackages)
def checkLiveServices(self, services, result):
is_redhat7_or_higher = False
is_redhat = False
if OSCheck.is_redhat_family():
is_redhat = True
if int(OSCheck.get_os_major_version()) >= 7:
is_redhat7_or_higher = True
for service in services:
svcCheckResult = {}
if "ntpd" in service and is_redhat7_or_higher:
svcCheckResult['name'] = "chronyd"
elif "chronyd" in service and is_redhat:
svcCheckResult['name'] = "ntpd"
else:
svcCheckResult['name'] = " or ".join(service)
svcCheckResult['status'] = "UNKNOWN"
svcCheckResult['desc'] = ""
try:
out = ""
err = ""
for serviceName in service:
sys_out, sys_err, code = self.getServiceStatus(serviceName)
if code == 0:
break
out += sys_out if len(out) == 0 else os.linesep + sys_out
err += sys_err if len(err) == 0 else os.linesep + sys_err
if 0 != code:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = out
if len(out) == 0:
svcCheckResult['desc'] = err
else:
svcCheckResult['status'] = "Healthy"
except Exception, e:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = repr(e)
result.append(svcCheckResult)
def get_ntp_service():
if OSCheck.is_redhat_family():
return "ntpd"
elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
return "ntp"
webhcat_server_win_service_name = "templeton"
else:
hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
hive_pid = 'hive-server.pid'
hive_metastore_pid = 'hive.pid'
hcat_pid_dir = config['configurations']['hive-env'][
'hcat_pid_dir'] #hcat_pid_dir
webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')
process_name = 'mysqld'
if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
daemon_name = 'mysql'
else:
if OSCheck.is_redhat_family() and OSCheck.get_os_major_version(
) == "7":
daemon_name = 'mariadb'
else:
daemon_name = 'mysqld'
# Security related/required params
hostname = config['hostname']
security_enabled = config['configurations']['cluster-env'][
'security_enabled']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hive_user = config['configurations']['hive-env']['hive_user']
webhcat_user = config['configurations']['hive-env']['webhcat_user']
def get_repo_dir():
if OSCheck.is_redhat_family():
return '/etc/yum.repos.d'
elif OSCheck.is_suse_family():
return '/etc/zypp/repos.d'
def get_ntp_service():
if OSCheck.is_redhat_family():
return "ntpd"
elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
return "ntp"
config = Script.get_config()
hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
hive_pid = 'hive-server.pid'
hive_metastore_pid = 'hive.pid'
hcat_pid_dir = config['configurations']['hive-env'][
'hcat_pid_dir'] #hcat_pid_dir
webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')
process_name = 'mysqld'
if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
daemon_name = 'mysql'
elif OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
daemon_name = 'mariadb'
else:
daemon_name = 'mysqld'
# Security related/required params
hostname = config['hostname']
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
tmp_dir = Script.get_tmp_dir()
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hive_user = config['configurations']['hive-env']['hive_user']
webhcat_user = config['configurations']['hive-env']['webhcat_user']
# default configuration directories
def get_repo_dir():
if OSCheck.is_redhat_family():
return '/etc/yum.repos.d'
elif OSCheck.is_suse_family():
return '/etc/zypp/repos.d'
