def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(
config['hostLevelParams']
['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if Script.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
install_windows_msi(config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], ["hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab"],
hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger(__name__)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def installAgent(projectVersion, ret=None):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = [
"zypper", "--no-gpg-checks", "install", "-y",
"ambari-agent-" + projectVersion
]
elif OSCheck.is_ubuntu_family():
# add * to end of version in case of some test releases
Command = [
"apt-get", "install", "-y", "--allow-unauthenticated",
"ambari-agent=" + projectVersion + "*"
]
elif OSCheck.is_windows_family():
packageParams = "/AmbariRoot:" + AMBARI_INSTALL_ROOT
Command = [
"cmd", "/c", "choco", "install", "-y", "ambari-agent",
"--version=" + projectVersion, "--params=\"" + packageParams + "\""
]
else:
Command = [
"yum", "-y", "install", "--nogpgcheck",
"ambari-agent-" + projectVersion
]
return execOsCommand(Command, tries=3, try_sleep=10, ret=ret)
def _build_web_query(self, alert_uri):
"""
Builds a URL out of the URI structure. If the URI is already a URL of
the form http[s]:// then this will return the URI as the URL; otherwise,
it will build the URL from the URI structure's elements
"""
# shortcut if the supplied URI starts with the information needed
string_uri = str(alert_uri.uri)
if string_uri.startswith('http://') or string_uri.startswith(
'https://'):
return alert_uri.uri
# start building the URL manually
host = BaseAlert.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
# maybe slightly realistic
port = 80
if alert_uri.is_ssl_enabled is True:
port = 443
# extract the port
try:
port = int(get_port_from_url(alert_uri.uri))
except:
pass
scheme = 'http'
if alert_uri.is_ssl_enabled is True:
scheme = 'https'
if OSCheck.is_windows_family():
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
host = resolve_address(host)
return "{0}://{1}:{2}".format(scheme, host, str(port))
def install_packages(self, env, exclude_packages=[]):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if not package['name'] in exclude_packages:
name = package['name']
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
self.set_version()
def resolve_address(address):
"""
Resolves address to proper one in special cases, for example 0.0.0.0 to 127.0.0.1 on windows os.
:param address: address to resolve
:return: resulting address
"""
if OSCheck.is_windows_family():
if address == '0.0.0.0':
return '127.0.0.1'
return address
def isAgentPackageAlreadyInstalled(projectVersion):
if OSCheck.is_ubuntu_family():
Command = ["bash", "-c", "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep " + projectVersion]
elif OSCheck.is_windows_family():
Command = ["cmd", "/c", "choco list ambari-agent --local-only | findstr ambari-agent | findstr " + projectVersion]
else:
Command = ["bash", "-c", "rpm -qa | grep ambari-agent-"+projectVersion]
ret = execOsCommand(Command)
res = False
if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
res = True
return res
def getAvailableAgentPackageVersions():
if OSCheck.is_suse_family():
Command = ["bash", "-c",
"zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
elif OSCheck.is_windows_family():
Command = ["cmd", "/c", "choco list ambari-agent --pre --all | findstr ambari-agent"]
elif OSCheck.is_ubuntu_family():
Command = ["bash", "-c",
"apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"]
else:
Command = ["bash", "-c",
"yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
return execOsCommand(Command)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# parse arguments
if len(sys.argv) < 7:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logging.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def get_ambari_repo_file_full_name():
if OSCheck.is_ubuntu_family():
ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
elif OSCheck.is_redhat_family():
ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
elif OSCheck.is_suse_family():
ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
elif OSCheck.is_windows_family():
ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
else:
raise Exception('Ambari repo file path not set for current OS.')
return ambari_repo_file
def get_ambari_repo_file_full_name():
if OSCheck.is_ubuntu_family():
ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
elif OSCheck.is_redhat_family():
ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
elif OSCheck.is_suse_family():
ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
elif OSCheck.is_windows_family():
ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
else:
raise Exception('Ambari repo file path not set for current OS.')
return ambari_repo_file
def installAgent(projectVersion, ret=None):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion]
elif OSCheck.is_ubuntu_family():
# add * to end of version in case of some test releases
Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"]
elif OSCheck.is_windows_family():
packageParams = "/AmbariRoot:" + AMBARI_INSTALL_ROOT
Command = ["cmd", "/c", "choco", "install", "-y", "ambari-agent", "--version=" + projectVersion, "--params=\"" + packageParams + "\""]
else:
Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
return execOsCommand(Command, tries=3, try_sleep=10, ret=ret)
def main(argv=None):
scriptDir = os.path.realpath(os.path.dirname(argv[0]))
onlyargs = argv[1:]
if len(onlyargs) < 3:
sys.stderr.write(
"Usage: <comma separated hosts> "
"<tmpdir for storage> <user> <sshPort> <sshkey_file> <agent setup script>"
" <ambari-server name> <cluster os type> <ambari version> <ambari port> <user_run_as> <passwordFile>\n"
)
sys.exit(2)
pass
#Parse the input
hostList = onlyargs[0].split(",")
bootdir = onlyargs[1]
user = onlyargs[2]
sshPort = onlyargs[3]
sshkey_file = onlyargs[4]
setupAgentFile = onlyargs[5]
ambariServer = onlyargs[6]
cluster_os_type = onlyargs[7]
ambariVersion = onlyargs[8]
server_port = onlyargs[9]
user_run_as = onlyargs[10]
passwordFile = onlyargs[11]
if not OSCheck.is_windows_family():
# ssh doesn't like open files
subprocess32.Popen(["chmod", "600", sshkey_file],
stdout=subprocess32.PIPE)
if passwordFile is not None and passwordFile != 'null':
subprocess32.Popen(["chmod", "600", passwordFile],
stdout=subprocess32.PIPE)
logging.info("BootStrapping hosts " + pprint.pformat(hostList) +
" using " + scriptDir + " cluster primary OS: " + cluster_os_type +
" with user '" + user + "'with ssh Port '" + sshPort + "' sshKey File " + sshkey_file + " password File " + passwordFile +\
" using tmp dir " + bootdir + " ambari: " + ambariServer +"; server_port: " + server_port +\
"; ambari version: " + ambariVersion+"; user_run_as: " + user_run_as)
sharedState = SharedState(user, sshPort, sshkey_file, scriptDir, bootdir,
setupAgentFile, ambariServer, cluster_os_type,
ambariVersion, server_port, user_run_as,
passwordFile)
pbootstrap = PBootstrap(hostList, sharedState)
pbootstrap.run()
return 0 # Hack to comply with current usage
def _build_web_query(self, alert_uri):
"""
Builds a URL out of the URI structure. If the URI is already a URL of
the form http[s]:// then this will return the URI as the URL; otherwise,
it will build the URL from the URI structure's elements
"""
# shortcut if the supplied URI starts with the information needed
string_uri = str(alert_uri.uri)
if string_uri.startswith('http://') or string_uri.startswith('https://'):
return alert_uri.uri
uri_path = None
if string_uri and string_uri != str(None):
uri_path = get_path_from_url(string_uri)
# start building the URL manually
host = BaseAlert.get_host_from_url(alert_uri.uri)
if host is None:
host = self.host_name
# maybe slightly realistic
port = 80
if alert_uri.is_ssl_enabled is True:
port = 443
# extract the port
try:
port = int(get_port_from_url(alert_uri.uri))
except:
pass
scheme = 'http'
if alert_uri.is_ssl_enabled is True:
scheme = 'https'
if OSCheck.is_windows_family():
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
host = resolve_address(host)
if uri_path:
return "{0}://{1}:{2}/{3}".format(scheme, host, str(port), uri_path)
else:
return "{0}://{1}:{2}".format(scheme, host, str(port))
def main(argv=None):
scriptDir = os.path.realpath(os.path.dirname(argv[0]))
onlyargs = argv[1:]
if len(onlyargs) < 3:
sys.stderr.write("Usage: <comma separated hosts> "
"<tmpdir for storage> <user> <sshkey_file> <agent setup script>"
" <ambari-server name> <cluster os type> <ambari version> <ambari port> <user_run_as> <passwordFile>\n")
sys.exit(2)
pass
#Parse the input
hostList = onlyargs[0].split(",")
bootdir = onlyargs[1]
user = onlyargs[2]
sshkey_file = onlyargs[3]
setupAgentFile = onlyargs[4]
ambariServer = onlyargs[5]
cluster_os_type = onlyargs[6]
ambariVersion = onlyargs[7]
server_port = onlyargs[8]
user_run_as = onlyargs[9]
passwordFile = onlyargs[10]
if not OSCheck.is_windows_family():
# ssh doesn't like open files
subprocess.Popen(["chmod", "600", sshkey_file], stdout=subprocess.PIPE)
if passwordFile is not None and passwordFile != 'null':
subprocess.Popen(["chmod", "600", passwordFile], stdout=subprocess.PIPE)
logging.info("BootStrapping hosts " + pprint.pformat(hostList) +
" using " + scriptDir + " cluster primary OS: " + cluster_os_type +
" with user '" + user + "' sshKey File " + sshkey_file + " password File " + passwordFile +\
" using tmp dir " + bootdir + " ambari: " + ambariServer +"; server_port: " + server_port +\
"; ambari version: " + ambariVersion+"; user_run_as: " + user_run_as)
sharedState = SharedState(user, sshkey_file, scriptDir, bootdir, setupAgentFile,
ambariServer, cluster_os_type, ambariVersion,
server_port, user_run_as, passwordFile)
pbootstrap = PBootstrap(hostList, sharedState)
pbootstrap.run()
return 0 # Hack to comply with current usage
def findNearestAgentPackageVersion(projectVersion):
if projectVersion == "":
projectVersion = " "
if OSCheck.is_suse_family():
Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion +
"' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "]
elif OSCheck.is_windows_family():
listPackagesCommand = ["cmd", "/c", "choco list ambari-agent --pre --all | findstr " + projectVersion + " > agentPackages.list"]
execOsCommand(listPackagesCommand)
Command = ["cmd", "/c", "powershell", "get-content agentPackages.list | select-object -last 1 | foreach-object {$_ -replace 'ambari-agent ', ''}"]
elif OSCheck.is_ubuntu_family():
if projectVersion == " ":
Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
else:
Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '" +
projectVersion + "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
else:
Command = ["bash", "-c", "yum -q list all ambari-agent | grep '" + projectVersion +
"' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "]
return execOsCommand(Command)
def isAgentPackageAlreadyInstalled(projectVersion):
if OSCheck.is_ubuntu_family():
Command = [
"bash", "-c",
"dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep "
+ projectVersion
]
elif OSCheck.is_windows_family():
Command = [
"cmd", "/c",
"choco list ambari-agent --local-only | findstr ambari-agent"
]
else:
Command = [
"bash", "-c", "rpm -qa | grep ambari-agent-" + projectVersion
]
ret = execOsCommand(Command)
res = False
if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
res = True
return res
def findNearestAgentPackageVersion(projectVersion):
if projectVersion == "":
projectVersion = " "
if OSCheck.is_suse_family():
Command = [
"bash", "-c",
"zypper --no-gpg-checks --non-interactive -q search -s --match-exact ambari-agent | grep '"
+ projectVersion +
"' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "
]
elif OSCheck.is_windows_family():
listPackagesCommand = [
"cmd", "/c", "choco list ambari-agent --pre --all | findstr " +
projectVersion + " > agentPackages.list"
]
execOsCommand(listPackagesCommand)
Command = [
"cmd", "/c", "powershell",
"get-content agentPackages.list | select-object -last 1 | foreach-object {$_ -replace 'ambari-agent ', ''}"
]
elif OSCheck.is_ubuntu_family():
if projectVersion == " ":
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"
]
else:
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '"
+ projectVersion +
"'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"
]
else:
Command = [
"bash", "-c",
"yum -q list all ambari-agent | grep '" + projectVersion +
"' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "
]
return execOsCommand(Command)
def getAvailableAgentPackageVersions():
if OSCheck.is_suse_family():
Command = [
"bash", "-c",
"zypper --no-gpg-checks --non-interactive -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
elif OSCheck.is_windows_family():
Command = [
"cmd", "/c",
"choco list ambari-agent --pre --all | findstr ambari-agent"
]
elif OSCheck.is_ubuntu_family():
Command = [
"bash", "-c",
"apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
else:
Command = [
"bash", "-c",
"yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"
]
return execOsCommand(Command)
def get_file_owner(file_full_name):
if OSCheck.is_windows_family():
return ""
else:
return pwd.getpwuid(os.stat(file_full_name).st_uid).pw_name
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
kerberos_keytab = None
if KERBEROS_KEYTAB in configurations:
kerberos_keytab = configurations[KERBEROS_KEYTAB]
kerberos_principal = None
if KERBEROS_PRINCIPAL in configurations:
kerberos_principal = configurations[KERBEROS_PRINCIPAL]
kerberos_principal = kerberos_principal.replace('_HOST', host_name)
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
env = Environment.get_instance()
# curl requires an integer timeout
curl_connection_timeout = int(connection_timeout)
url_response, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal,
query, "nm_health_alert", None, False, "NodeManager Health", smokeuser,
connection_timeout=curl_connection_timeout)
json_response = json.loads(url_response)
else:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query, timeout=connection_timeout)
json_response = json.loads(url_response.read())
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError))
return (RESULT_CODE_CRITICAL, [label])
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script argument for forcing https protocol
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if self.command_name == "start" and not self.is_hook():
self.pre_start()
method(env)
if self.command_name == "start" and not self.is_hook():
self.post_start()
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
from resource_management.libraries.functions import lzo_utils
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from functions import calc_xmn_from_xms
from resource_management import *
import status_params
from ambari_commons import OSCheck
if OSCheck.is_windows_family():
from params_windows import *
else:
from params_linux import *
# server configurations
config = Script.get_config()
exec_tmp_dir = Script.get_tmp_dir()
# AMBARI_METRICS data
ams_pid_dir = status_params.ams_collector_pid_dir
ams_collector_script = "/usr/sbin/ambari-metrics-collector"
ams_collector_pid_dir = status_params.ams_collector_pid_dir
ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
ams_collector_host_single = ams_collector_hosts[0] # TODO cardinality is 1+ so we can have more than one host
metric_collector_port = default("/configurations/ams-site/timeline.metrics.service.webapp.address", "0.0.0.0:6188")
def get_file_owner(file_full_name):
if OSCheck.is_windows_family():
return ""
else:
return pwd.getpwuid(os.stat(file_full_name).st_uid).pw_name
def current_user():
if OSCheck.is_windows_family():
return None
else:
return pwd.getpwuid(os.geteuid())[0]
def _collect(self):
# can be parameterized or static
# if not parameterized, this will return the static value
uri_value = self._get_configuration_value(self.uri)
host_not_specified = False
if uri_value is None:
host_not_specified = True
uri_value = self.host_name
logger.debug("[Alert][{0}] Setting the URI to this host since it wasn't specified".format(
self.get_name()))
# in some cases, a single property is a comma-separated list like
# host1:8080,host2:8081,host3:8083
uri_value_array = uri_value.split(',')
if len(uri_value_array) > 1:
for item in uri_value_array:
if self.host_name in item:
uri_value = item
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Extracted {1} as the host name while parsing the CSV URI {2}".format(
self.get_name(), uri_value, str(uri_value_array)))
break
host = BaseAlert.get_host_from_url(uri_value)
if host is None or host == "localhost" or host == "0.0.0.0":
host = self.host_name
host_not_specified = True
hosts = [host]
# If host is not specified in the uri, hence we are using current host name
# then also add public host name as a fallback.
if host_not_specified and host.lower() == self.host_name.lower() \
and self.host_name.lower() != self.public_host_name.lower():
hosts.append(self.public_host_name)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] List of hosts = {1}".format(self.get_name(), hosts))
try:
port = int(get_port_from_url(uri_value))
except:
if self.default_port is None:
label = 'Unable to determine port from URI {0}'.format(uri_value)
return (self.RESULT_UNKNOWN, [label])
port = self.default_port
exceptions = []
for host in hosts:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Checking {1} on port {2}".format(
self.get_name(), host, str(port)))
s = None
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(self.critical_timeout)
if OSCheck.is_windows_family():
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
host = resolve_address(host)
start_time = time.time()
s.connect((host, port))
if self.socket_command is not None:
s.sendall(self.socket_command)
data = s.recv(1024)
if self.socket_command_response is not None and data != self.socket_command_response:
raise Exception("Expected response {0}, Actual response {1}".format(
self.socket_command_response, data))
end_time = time.time()
milliseconds = end_time - start_time
seconds = milliseconds / 1000.0
# not sure why this happens sometimes, but we don't always get a
# socket exception if the connect() is > than the critical threshold
if seconds >= self.critical_timeout:
return (self.RESULT_CRITICAL, ['Socket Timeout', host, port])
result = self.RESULT_OK
if seconds >= self.warning_timeout:
result = self.RESULT_WARNING
return (result, [seconds, port])
except Exception as e:
exceptions.append(e)
finally:
if s is not None:
try:
s.close()
except:
# no need to log a close failure
pass
if exceptions:
return (self.RESULT_CRITICAL, [str(exceptions[0]), hosts[0], port])
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from functions import calc_xmn_from_xms
from resource_management import *
import status_params
from ambari_commons import OSCheck
if OSCheck.is_windows_family():
from params_windows import *
else:
from params_linux import *
# server configurations
config = Script.get_config()
exec_tmp_dir = Script.get_tmp_dir()
def get_combined_memory_mb(value1, value2):
try:
part1 = int(value1.strip()[:-1]) if value1.lower().strip(
)[-1:] == 'm' else int(value1)
part2 = int(value2.strip()[:-1]) if value2.lower().strip(
)[-1:] == 'm' else int(value2)
return str(part1 + part2) + 'm'
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code,
['There were no configurations supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme, uri)
try:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query, timeout=connection_timeout)
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError))
return (RESULT_CODE_CRITICAL, [label])
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
executable_paths = None
if EXECUTABLE_SEARCH_PATHS in configurations:
executable_paths = configurations[EXECUTABLE_SEARCH_PATHS]
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
kerberos_keytab = None
if KERBEROS_KEYTAB in configurations:
kerberos_keytab = configurations[KERBEROS_KEYTAB]
kerberos_principal = None
if KERBEROS_PRINCIPAL in configurations:
kerberos_principal = configurations[KERBEROS_PRINCIPAL]
kerberos_principal = kerberos_principal.replace('_HOST', host_name)
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
env = Environment.get_instance()
# curl requires an integer timeout
curl_connection_timeout = int(connection_timeout)
url_response, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal,
query, "nm_health_alert", executable_paths, False, "NodeManager Health", smokeuser,
connection_timeout=curl_connection_timeout)
json_response = json.loads(url_response)
else:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query, timeout=connection_timeout)
json_response = json.loads(url_response.read())
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError), traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
try:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query, timeout=connection_timeout)
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError))
return (RESULT_CODE_CRITICAL, [label])
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if parameters is None:
return (result_code,
['There were no parameters supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if NODEMANAGER_HTTP_ADDRESS_KEY in parameters:
http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters:
https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in parameters:
http_policy = parameters[YARN_HTTP_POLICY_KEY]
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
try:
query = "{0}://{1}/ws/v1/node/info".format(scheme, uri)
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query)
except:
label = CRITICAL_CONNECTION_MESSAGE.format(uri)
return (RESULT_CODE_CRITICAL, [label])
# URL response received, parse it
try:
json_response = json.loads(url_response.read())
node_healthy = json_response['nodeInfo']['nodeHealthy']
# convert boolean to string
node_healthy = str(node_healthy)
except:
return (RESULT_CODE_CRITICAL, [query])
# proper JSON received, compare against known value
if node_healthy.lower() == 'true':
result_code = RESULT_CODE_OK
label = OK_MESSAGE
else:
result_code = RESULT_CODE_CRITICAL
label = CRITICAL_NODEMANAGER_STATUS_MESSAGE.format(node_healthy)
return (result_code, [label])
def _collect(self):
# can be parameterized or static
# if not parameterized, this will return the static value
uri_value = self._get_configuration_value(self.uri)
if uri_value is None:
uri_value = self.host_name
logger.debug("[Alert][{0}] Setting the URI to this host since it wasn't specified".format(
self.get_name()))
# in some cases, a single property is a comma-separated list like
# host1:8080,host2:8081,host3:8083
uri_value_array = uri_value.split(',')
if len(uri_value_array) > 1:
for item in uri_value_array:
if self.host_name in item:
uri_value = item
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Extracted {1} as the host name while parsing the CSV URI {2}".format(
self.get_name(), uri_value, str(uri_value_array)))
break
host = BaseAlert.get_host_from_url(uri_value)
if host is None:
host = self.host_name
try:
port = int(get_port_from_url(uri_value))
except:
if self.default_port is None:
label = 'Unable to determine port from URI {0}'.format(uri_value)
return (self.RESULT_UNKNOWN, [label])
port = self.default_port
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Checking {1} on port {2}".format(
self.get_name(), host, str(port)))
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(self.critical_timeout)
if OSCheck.is_windows_family():
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
host = resolve_address(host)
start_time = time.time()
s.connect((host, port))
end_time = time.time()
milliseconds = end_time - start_time
seconds = milliseconds / 1000.0
# not sure why this happens sometimes, but we don't always get a
# socket exception if the connect() is > than the critical threshold
if seconds >= self.critical_timeout:
return (self.RESULT_CRITICAL, ['Socket Timeout', host, port])
result = self.RESULT_OK
if seconds >= self.warning_timeout:
result = self.RESULT_WARNING
return (result, [seconds, port])
except Exception as e:
return (self.RESULT_CRITICAL, [str(e), host, port])
finally:
if s is not None:
try:
s.close()
except:
# no need to log a close failure
pass
