def test_get_os_type(self, mock_exists, mock_linux_distribution):
# 1 - Any system
mock_exists.return_value = False
mock_linux_distribution.return_value = ('my_os', '', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'my_os')
# 2 - Negative case
mock_linux_distribution.return_value = ('', 'aaaa', 'bbbbb')
try:
result = OSCheck.get_os_type()
self.fail("Should throw exception in OSCheck.get_os_type()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os type. Exiting...", str(e))
pass
# 3 - path exist: '/etc/oracle-release'
mock_exists.return_value = True
mock_linux_distribution.return_value = ('some_os', '', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'oraclelinux')
# 4 - Common system
mock_exists.return_value = False
mock_linux_distribution.return_value = ('CenToS', '', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'centos')
def test_is_redhat_family(self, get_os_family_mock):
get_os_family_mock.return_value = "redhat"
self.assertEqual(OSCheck.is_redhat_family(), True)
get_os_family_mock.return_value = "troll_os"
self.assertEqual(OSCheck.is_redhat_family(), False)
def findNearestAgentPackageVersion(projectVersion):
if projectVersion == "":
projectVersion = " "
if OSCheck.is_suse_family():
Command = [
"bash", "-c",
"zypper -q search -s --match-exact ambari-agent | grep '" +
projectVersion +
"' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "
]
elif OSCheck.is_debian_family():
if projectVersion == " ":
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"
]
else:
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '"
+ projectVersion +
"'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z\d]*//'"
]
else:
Command = [
"bash", "-c",
"yum -q list all ambari-agent | grep '" + projectVersion +
"' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "
]
return execOsCommand(Command)
def test_get_os_major_version(self, mock_linux_distribution):
# 1
mock_linux_distribution.return_value = ('', '123.45.67', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '123')
# 2
mock_linux_distribution.return_value = ('Suse', '11', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '11')
def test_get_os_version(self, mock_linux_distribution):
# 1 - Any system
mock_linux_distribution.return_value = ('', '123.45', '')
result = OSCheck.get_os_version()
self.assertEquals(result, '123.45')
# 2 - Negative case
mock_linux_distribution.return_value = ('ssss', '', 'ddddd')
try:
result = OSCheck.get_os_version()
self.fail("Should throw exception in OSCheck.get_os_version()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os version. Exiting...", str(e))
pass
def checkLiveServices(self, services, result):
osType = OSCheck.get_os_family()
for service in services:
svcCheckResult = {}
if isinstance(service, dict):
serviceName = service[osType]
else:
serviceName = service
service_check_live = shlex.split(self.SERVICE_STATUS_CMD)
service_check_live[1] = serviceName
svcCheckResult['name'] = serviceName
svcCheckResult['status'] = "UNKNOWN"
svcCheckResult['desc'] = ""
try:
osStat = subprocess.Popen(service_check_live,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = osStat.communicate()
if 0 != osStat.returncode:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = out
if len(out) == 0:
svcCheckResult['desc'] = err
else:
svcCheckResult['status'] = "Healthy"
except Exception, e:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = repr(e)
result.append(svcCheckResult)
def find_repo_files_for_repos(self, repoNames):
repoFiles = []
osType = OSCheck.get_os_family()
repoNameList = []
for repoName in repoNames:
if len(repoName.strip()) > 0:
repoNameList.append("[" + repoName + "]")
repoNameList.append("name=" + repoName)
if repoNameList:
# get list of files
if osType == 'suse':
fileList = self.get_files_in_dir(REPO_PATH_SUSE)
elif osType == "redhat":
fileList = self.get_files_in_dir(REPO_PATH_RHEL)
else:
logger.warn(
"Unsupported OS type, cannot get repository location.")
return []
if fileList:
for filePath in fileList:
with open(filePath, 'r') as file:
content = file.readline()
while (content != ""):
for repoName in repoNameList:
if content.find(
repoName
) == 0 and filePath not in repoFiles:
repoFiles.append(filePath)
break
content = file.readline()
return repoFiles
def test_get_os_release_name(self, mock_linux_distribution):
# 1 - Any system
mock_linux_distribution.return_value = ('', '', 'MY_NEW_RELEASE')
result = OSCheck.get_os_release_name()
self.assertEquals(result, 'my_new_release')
# 2 - Negative case
mock_linux_distribution.return_value = ('aaaa', 'bbbb', '')
try:
result = OSCheck.get_os_release_name()
self.fail("Should throw exception in OSCheck.get_os_release_name()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os release name. Exiting...", str(e))
pass
def os_version(self):
"""
Example return value:
"6.3" for "Centos 6.3"
In case cannot detect --> Fail
"""
return OSCheck.get_os_version()
def os_family(self):
"""
Return values:
redhat, debian, suse
In case cannot detect raises exception
"""
return OSCheck.get_os_family()
def getAvaliableAgentPackageVersions():
if OSCheck.is_suse_family():
Command = [
"bash", "-c",
"zypper -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
elif OSCheck.is_debian_family():
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
else:
Command = [
"bash", "-c",
"yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
return execOsCommand(Command)
def installAgent(projectVersion):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = ["zypper", "install", "-y", "ambari-agent-" + projectVersion]
elif OSCheck.is_debian_family():
# add * to end of version in case of some test releases
Command = [
"apt-get", "install", "-y", "--force-yes",
"ambari-agent=" + projectVersion + "*"
]
else:
Command = [
"yum", "-y", "install", "--nogpgcheck",
"ambari-agent-" + projectVersion
]
return execOsCommand(Command)
def main(argv=None):
# Same logic that was in "os_type_check.sh"
if len(sys.argv) != 2:
print "Usage: <cluster_os>"
raise Exception("Error in number of arguments. Usage: <cluster_os>")
pass
cluster_os = sys.argv[1]
current_os = OSCheck.get_os_family() + OSCheck.get_os_major_version()
# If agent/server have the same {"family","main_version"} - then ok.
print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % (
cluster_os, current_os)
if current_os == cluster_os:
sys.exit(0)
else:
raise Exception(
"Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host."
)
def os_type(self):
"""
Return values:
redhat, fedora, centos, oraclelinux, ascendos,
amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm,
ubuntu, debian, sles, sled, opensuse, suse ... and others
In case cannot detect raises exception.
"""
return OSCheck.get_os_type()
def isAgentPackageAlreadyInstalled(projectVersion):
if OSCheck.is_debian_family():
Command = [
"bash", "-c", "dpkg -s ambari-agent 2>&1|grep 'Version\:'|grep " +
projectVersion
]
else:
Command = [
"bash", "-c", "rpm -qa | grep ambari-agent-" + projectVersion
]
ret = execOsCommand(Command)
res = False
if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
res = True
return res
def do_erase_packages(self, packageList):
packageStr = None
if packageList:
packageStr = ' '.join(packageList)
logger.debug("Erasing packages: " + packageStr)
if packageStr is not None and packageStr:
os_name = OSCheck.get_os_family()
command = ''
if os_name == 'suse':
command = PACKAGE_ERASE_CMD_SUSE.format(packageStr)
elif os_name == 'redhat':
command = PACKAGE_ERASE_CMD_RHEL.format(packageStr)
else:
logger.warn("Unsupported OS type, cannot remove package.")
if command != '':
logger.debug('Executing: ' + str(command))
(returncode, stdoutdata,
stderrdata) = self.run_os_command(command)
if returncode != 0:
logger.warn("Erasing packages failed: " + stderrdata)
else:
logger.info("Erased packages successfully.\n" + stdoutdata)
return 0
def test_get_os_family(self, mock_exists, mock_linux_distribution):
# 1 - Any system
mock_exists.return_value = False
mock_linux_distribution.return_value = ('MY_os', '', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'my_os')
# 2 - Redhat
mock_exists.return_value = False
mock_linux_distribution.return_value = ('Centos Linux', '', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'redhat')
# 3 - Debian
mock_exists.return_value = False
mock_linux_distribution.return_value = ('Ubuntu', '', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'debian')
# 4 - Suse
mock_exists.return_value = False
mock_linux_distribution.return_value = (
'suse linux enterprise server', '', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'suse')
mock_exists.return_value = False
mock_linux_distribution.return_value = ('SLED', '', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'suse')
# 5 - Negative case
mock_linux_distribution.return_value = ('', '111', '2222')
try:
result = OSCheck.get_os_family()
self.fail("Should throw exception in OSCheck.get_os_family()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os type. Exiting...", str(e))
pass
def getOsFamily(self):
return OSCheck.get_os_family()
class HostInfo:
# List of project names to be used to find alternatives folders etc.
DEFAULT_PROJECT_NAMES = [
"hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
"oozie", "sqoop", "hue", "zookeeper", "mapred", "hdfs", "flume",
"storm", "hive-hcatalog", "tez", "falcon", "ambari_qa",
"hadoop_deploy", "rrdcached", "hcat", "ambari-qa", "sqoop-ambari-qa",
"sqoop-ambari_qa", "webhcat", "hadoop-hdfs", "hadoop-yarn",
"hadoop-mapreduce"
]
# List of live services checked for on the host, takes a map of plan strings
DEFAULT_LIVE_SERVICES = [{
"redhat": "ntpd",
"suse": "ntp",
"debian": "ntp"
}]
# Set of default users (need to be replaced with the configured user names)
DEFAULT_USERS = [
"nagios", "hive", "ambari-qa", "oozie", "hbase", "hcat", "mapred",
"hdfs", "rrdcached", "zookeeper", "flume", "sqoop", "sqoop2", "hue",
"yarn"
]
# Filters used to identify processed
PROC_FILTER = ["hadoop", "zookeeper"]
# Additional path patterns to find existing directory
DIRNAME_PATTERNS = ["/tmp/hadoop-", "/tmp/hsperfdata_"]
# Default set of directories that are checked for existence of files and folders
DEFAULT_DIRS = [
"/etc", "/var/run", "/var/log", "/usr/lib", "/var/lib", "/var/tmp",
"/tmp", "/var", "/hadoop"
]
# Packages that are used to find repos (then repos are used to find other packages)
PACKAGES = [
"hadoop", "zookeeper", "webhcat", "*-manager-server-db",
"*-manager-daemons"
]
# Additional packages to look for (search packages that start with these)
ADDITIONAL_PACKAGES = [
"rrdtool", "rrdtool-python", "nagios", "ganglia", "gmond", "gweb",
"libconfuse", "ambari-log4j", "hadoop", "zookeeper", "oozie", "webhcat"
]
# ignore packages from repos whose names start with these strings
IGNORE_PACKAGES_FROM_REPOS = ["ambari", "installed"]
# ignore required packages
IGNORE_PACKAGES = ["epel-release"]
# ignore repos from the list of repos to be cleaned
IGNORE_REPOS = ["ambari", "HDP-UTILS"]
# default timeout for async invoked processes
TIMEOUT_SECONDS = 60
RESULT_UNAVAILABLE = "unable_to_determine"
OS_FAMILY = OSCheck.get_os_family()
OS_UBUNTU_DEBIAN = 'debian'
# service cmd
SERVICE_CMD = "/sbin/service"
FIREWALL_SERVICE_NAME = "iptables"
# on ubuntu iptables service is called ufw
if OS_FAMILY == OS_UBUNTU_DEBIAN:
SERVICE_CMD = "/usr/sbin/service"
FIREWALL_SERVICE_NAME = "ufw"
FIREWALL_STATUS_CMD = "%s %s status" % (SERVICE_CMD, FIREWALL_SERVICE_NAME)
DEFAULT_SERVICE_NAME = "ntpd"
SERVICE_STATUS_CMD = "%s %s status" % (SERVICE_CMD, DEFAULT_SERVICE_NAME)
event = threading.Event()
current_umask = -1
def __init__(self, config=None):
self.packages = PackagesAnalyzer()
self.reportFileHandler = HostCheckReportFileHandler(config)
def dirType(self, path):
if not os.path.exists(path):
return 'not_exist'
elif os.path.islink(path):
return 'sym_link'
elif os.path.isdir(path):
return 'directory'
elif os.path.isfile(path):
return 'file'
return 'unknown'
def hadoopVarRunCount(self):
if not os.path.exists('/var/run/hadoop'):
return 0
pids = glob.glob('/var/run/hadoop/*/*.pid')
return len(pids)
def hadoopVarLogCount(self):
if not os.path.exists('/var/log/hadoop'):
return 0
logs = glob.glob('/var/log/hadoop/*/*.log')
return len(logs)
def etcAlternativesConf(self, projects, etcResults):
if not os.path.exists('/etc/alternatives'):
return []
projectRegex = "'" + '|'.join(projects) + "'"
files = [
f for f in os.listdir('/etc/alternatives')
if re.match(projectRegex, f)
]
for conf in files:
result = {}
filePath = os.path.join('/etc/alternatives', conf)
if os.path.islink(filePath):
realConf = os.path.realpath(filePath)
result['name'] = conf
result['target'] = realConf
etcResults.append(result)
def checkLiveServices(self, services, result):
osType = OSCheck.get_os_family()
for service in services:
svcCheckResult = {}
if isinstance(service, dict):
serviceName = service[osType]
else:
serviceName = service
service_check_live = shlex.split(self.SERVICE_STATUS_CMD)
service_check_live[1] = serviceName
svcCheckResult['name'] = serviceName
svcCheckResult['status'] = "UNKNOWN"
svcCheckResult['desc'] = ""
try:
osStat = subprocess.Popen(service_check_live,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = osStat.communicate()
if 0 != osStat.returncode:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = out
if len(out) == 0:
svcCheckResult['desc'] = err
else:
svcCheckResult['status'] = "Healthy"
except Exception, e:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = repr(e)
result.append(svcCheckResult)
def getOperatingSystem(self):
return OSCheck.get_os_family()
def os_release_name(self):
"""
For Ubuntu 12.04:
precise
"""
return OSCheck.get_os_release_name()
import re
import time
import subprocess
import threading
import shlex
import platform
from PackagesAnalyzer import PackagesAnalyzer
from HostCheckReportFileHandler import HostCheckReportFileHandler
from Hardware import Hardware
from common_functions import OSCheck, OSConst
import socket
logger = logging.getLogger()
# OS info
OS_VERSION = OSCheck().get_os_major_version()
OS_TYPE = OSCheck.get_os_type()
OS_FAMILY = OSCheck.get_os_family()
# service cmd
SERVICE_CMD = "/sbin/service"
# on ubuntu iptables service is called ufw
if OS_FAMILY == OSConst.DEBIAN_FAMILY:
SERVICE_CMD = "/usr/sbin/service"
class HostInfo:
# List of project names to be used to find alternatives folders etc.
DEFAULT_PROJECT_NAMES = [
"hadoop*", "hadoop", "hbase", "hcatalog", "hive", "ganglia", "nagios",
def getOperatingSystemRelease(self):
return OSCheck.get_os_version()
def getOperatingSystem(self):
return OSCheck.get_os_type()
def register(self, dict, componentsMapped=True, commandsInProgress=True):
dict['hostHealth'] = {}
java = []
self.javaProcs(java)
dict['hostHealth']['activeJavaProcs'] = java
liveSvcs = []
self.checkLiveServices(self.DEFAULT_LIVE_SERVICES, liveSvcs)
dict['hostHealth']['liveServices'] = liveSvcs
dict['umask'] = str(self.getUMask())
# detailed host check is not available for Suse
isSuse = 'suse' == OSCheck.get_os_family()
dict['iptablesIsRunning'] = self.checkIptables()
dict['reverseLookup'] = self.checkReverseLookup()
# If commands are in progress or components are already mapped to this host
# Then do not perform certain expensive host checks
if componentsMapped or commandsInProgress or isSuse:
dict['existingRepos'] = [self.RESULT_UNAVAILABLE]
dict['installedPackages'] = []
dict['alternatives'] = []
dict['stackFoldersAndFiles'] = []
dict['existingUsers'] = []
else:
etcs = []
self.etcAlternativesConf(self.DEFAULT_PROJECT_NAMES, etcs)
dict['alternatives'] = etcs
existingUsers = []
self.checkUsers(self.DEFAULT_USERS, existingUsers)
dict['existingUsers'] = existingUsers
dirs = []
self.checkFolders(self.DEFAULT_DIRS, self.DEFAULT_PROJECT_NAMES,
existingUsers, dirs)
dict['stackFoldersAndFiles'] = dirs
installedPackages = []
availablePackages = []
self.packages.allInstalledPackages(installedPackages)
self.packages.allAvailablePackages(availablePackages)
repos = []
self.packages.getInstalledRepos(
self.PACKAGES, installedPackages + availablePackages,
self.IGNORE_PACKAGES_FROM_REPOS, repos)
packagesInstalled = self.packages.getInstalledPkgsByRepo(
repos, self.IGNORE_PACKAGES, installedPackages)
additionalPkgsInstalled = self.packages.getInstalledPkgsByNames(
self.ADDITIONAL_PACKAGES, installedPackages)
allPackages = list(set(packagesInstalled +
additionalPkgsInstalled))
dict['installedPackages'] = self.packages.getPackageDetails(
installedPackages, allPackages)
repos = self.getReposToRemove(repos, self.IGNORE_REPOS)
dict['existingRepos'] = repos
self.reportFileHandler.writeHostCheckFile(dict)
pass
# The time stamp must be recorded at the end
dict['hostHealth']['agentTimeStampAtReporting'] = int(time.time() *
1000)
pass
