def is_ubuntu_family(self, get_os_family_mock):
get_os_family_mock.return_value = "ubuntu"
self.assertEqual(OSCheck.is_ubuntu_family(), True)
get_os_family_mock.return_value = "troll_os"
self.assertEqual(OSCheck.is_ubuntu_family(), False)
def test_is_redhat_family(self, get_os_family_mock):
get_os_family_mock.return_value = "redhat"
self.assertEqual(OSCheck.is_redhat_family(), True)
get_os_family_mock.return_value = "troll_os"
self.assertEqual(OSCheck.is_redhat_family(), False)
def find_repo_files_for_repos(self, repoNames):
repoFiles = []
osType = OSCheck.get_os_family()
repoNameList = []
for repoName in repoNames:
if len(repoName.strip()) > 0:
repoNameList.append("[" + repoName + "]")
repoNameList.append("name=" + repoName)
if repoNameList:
# get list of files
if OSCheck.is_suse_family():
fileList = self.get_files_in_dir(REPO_PATH_SUSE)
elif OSCheck.is_redhat_family():
fileList = self.get_files_in_dir(REPO_PATH_RHEL)
elif OSCheck.is_ubuntu_family():
fileList = self.get_files_in_dir(REPO_PATH_UBUNTU)
else:
logger.warn("Unsupported OS type, cannot get repository location.")
return []
if fileList:
for filePath in fileList:
with open(filePath, 'r') as file:
content = file.readline()
while (content != "" ):
for repoName in repoNameList:
if content.find(repoName) == 0 and filePath not in repoFiles:
repoFiles.append(filePath)
break;
content = file.readline()
return repoFiles
def get_postgre_hba_dir(OS_FAMILY):
"""Return postgre hba dir location depends on OS.
Also depends on version of postgres creates symlink like postgresql-->postgresql-9.3
1) /etc/rc.d/init.d/postgresql --> /etc/rc.d/init.d/postgresql-9.3
2) /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
"""
if OSCheck.is_ubuntu_family():
# Like: /etc/postgresql/9.1/main/
return os.path.join(get_pg_hba_init_files(), get_ubuntu_pg_version(),
"main")
elif OSCheck.is_redhat_family() and int(OSCheck.get_os_major_version()) >= 7:
return PG_HBA_ROOT_DEFAULT
else:
if not os.path.isfile(get_pg_hba_init_files()):
# Link: /etc/init.d/postgresql --> /etc/init.d/postgresql-9.1
os.symlink(glob.glob(get_pg_hba_init_files() + '*')[0],
get_pg_hba_init_files())
# Get postgres_data location (default: /var/lib/pgsql/data)
cmd = "alias exit=return; source " + get_pg_hba_init_files() + " status &>/dev/null; echo $PGDATA"
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(PG_HBA_ROOT, err) = p.communicate()
if PG_HBA_ROOT and len(PG_HBA_ROOT.strip()) > 0:
return PG_HBA_ROOT.strip()
else:
return PG_HBA_ROOT_DEFAULT
def get_pg_hba_init_files():
if OSCheck.is_ubuntu_family():
return '/etc/postgresql'
elif OSCheck.is_redhat_family():
return '/etc/rc.d/init.d/postgresql'
elif OSCheck.is_suse_family():
return '/etc/init.d/postgresql'
else:
raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def getFirewallObject(self):
if OSCheck.is_ubuntu_family():
return UbuntuFirewallChecks()
elif self.OS_TYPE == OSConst.OS_FEDORA and int(self.OS_VERSION) >= 18:
return Fedora18FirewallChecks()
elif OSCheck.is_suse_family():
return SuseFirewallChecks()
else:
return FirewallChecks()
def get_erase_cmd():
if OSCheck.is_redhat_family():
return "yum erase -y {0}"
elif OSCheck.is_suse_family():
return "zypper -n -q remove {0}"
elif OSCheck.is_ubuntu_family():
return "/usr/bin/apt-get -y -q remove {0}"
else:
raise Exception("Unsupported OS family '{0}', cannot remove package. ".format(OSCheck.get_os_family()))
def getRepoDir(self):
if OSCheck.is_redhat_family():
return "/etc/yum.repos.d"
elif OSCheck.is_suse_family():
return "/etc/zypp/repos.d"
elif OSCheck.is_ubuntu_family():
return "/etc/apt/sources.list.d"
else:
raise Exception("Unsupported OS family '{0}'".format(OSCheck.get_os_family()))
def getAvaliableAgentPackageVersions():
if OSCheck.is_suse_family():
Command = ["bash", "-c",
"zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep ambari-agent | sed -re 's/\s+/ /g' | cut -d '|' -f 4 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
elif OSCheck.is_ubuntu_family():
Command = ["bash", "-c",
"apt-cache -q show ambari-agent|grep 'Version\:'|cut -d ' ' -f 2| tr '\\n' ', '|sed -s 's/[-|~][A-Za-z0-9]*//g'"]
else:
Command = ["bash", "-c",
"yum -q list all ambari-agent | grep -E '^ambari-agent' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | tr '\\n' ', ' | sed -s 's/[-|~][A-Za-z0-9]*//g'"]
return execOsCommand(Command)
def installAgent(projectVersion):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion]
elif OSCheck.is_ubuntu_family():
# add * to end of version in case of some test releases
Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"]
else:
Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
return execOsCommand(Command, tries=3, try_sleep=10)
def installAgent(projectVersion):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = ["zypper", "install", "-y", "ambari-agent-" + projectVersion]
elif OSCheck.is_debian_family():
# add * to end of version in case of some test releases
Command = ["apt-get", "install", "-y", "--force-yes", "ambari-agent=" + projectVersion + "*"]
else:
Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
return execOsCommand(Command)
def test_get_os_major_version(self, mock_linux_distribution):
# 1
mock_linux_distribution.return_value = ('', '123.45.67', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '123')
# 2
mock_linux_distribution.return_value = ('Suse', '11', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '11')
def test_get_os_major_version(self, mock_linux_distribution):
# 1
mock_linux_distribution.return_value = ("", "123.45.67", "")
result = OSCheck.get_os_major_version()
self.assertEquals(result, "123")
# 2
mock_linux_distribution.return_value = ("Suse", "11", "")
result = OSCheck.get_os_major_version()
self.assertEquals(result, "11")
def isAgentPackageAlreadyInstalled(projectVersion):
if OSCheck.is_ubuntu_family():
Command = ["bash", "-c", "dpkg-query -W -f='${Status} ${Version}\n' ambari-agent | grep -v deinstall | grep " + projectVersion]
elif OSCheck.is_windows_family():
Command = ["cmd", "/c", "choco list ambari-agent --local-only | findstr ambari-agent | findstr " + projectVersion]
else:
Command = ["bash", "-c", "rpm -qa | grep ambari-agent-"+projectVersion]
ret = execOsCommand(Command)
res = False
if ret["exitstatus"] == 0 and ret["log"][0].strip() != "":
res = True
return res
def test_aliases(self, mock_linux_distribution):
OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5'
OSConst.OS_FAMILY_COLLECTION.append({
'name': 'aliased_os_family',
'os_list': ["aliased_os"]
})
mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '')
self.assertEquals(OSCheck.get_os_type(), 'aliased_os')
self.assertEquals(OSCheck.get_os_major_version(), '5')
self.assertEquals(OSCheck.get_os_version(), '5.45.67')
self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family')
def get_ambari_repo_file_full_name():
if OSCheck.is_ubuntu_family():
ambari_repo_file = "/etc/apt/sources.list.d/ambari.list"
elif OSCheck.is_redhat_family():
ambari_repo_file = "/etc/yum.repos.d/ambari.repo"
elif OSCheck.is_suse_family():
ambari_repo_file = "/etc/zypp/repos.d/ambari.repo"
elif OSCheck.is_windows_family():
ambari_repo_file = os.path.join(os.environ[ChocolateyConsts.CHOCOLATEY_INSTALL_VAR_NAME],
ChocolateyConsts.CHOCOLATEY_CONFIG_DIR, ChocolateyConsts.CHOCOLATEY_CONFIG_FILENAME)
else:
raise Exception('Ambari repo file path not set for current OS.')
return ambari_repo_file
def getTransparentHugePage(self):
thp_regex = "\[(.+)\]"
file_name = None
if OSCheck.is_ubuntu_family():
file_name = self.THP_FILE_UBUNTU
elif OSCheck.is_redhat_family():
file_name = self.THP_FILE_REDHAT
if file_name and os.path.isfile(file_name):
with open(file_name) as f:
file_content = f.read()
return re.search(thp_regex, file_content).groups()[0]
else:
return ""
def installAgent(projectVersion, ret=None):
""" Run install and make sure the agent install alright """
# The command doesn't work with file mask ambari-agent*.rpm, so rename it on agent host
if OSCheck.is_suse_family():
Command = ["zypper", "--no-gpg-checks", "install", "-y", "ambari-agent-" + projectVersion]
elif OSCheck.is_ubuntu_family():
# add * to end of version in case of some test releases
Command = ["apt-get", "install", "-y", "--allow-unauthenticated", "ambari-agent=" + projectVersion + "*"]
elif OSCheck.is_windows_family():
packageParams = "/AmbariRoot:" + AMBARI_INSTALL_ROOT
Command = ["cmd", "/c", "choco", "install", "-y", "ambari-agent", "--version=" + projectVersion, "--params=\"" + packageParams + "\""]
else:
Command = ["yum", "-y", "install", "--nogpgcheck", "ambari-agent-" + projectVersion]
return execOsCommand(Command, tries=3, try_sleep=10, ret=ret)
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(config['hostLevelParams']['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(config['hostLevelParams']['agent_stack_retry_count'])
if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if Script.check_package_condition(package):
name = self.format_package_name(package['name'])
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
install_windows_msi(config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], ["hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab", "hdp-2.3.0.0-01.cab"],
hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def perform_prestart_checks(expected_hostname):
# Check if current hostname is equal to expected one (got from the server
# during bootstrap.
global config
if expected_hostname is not None:
current_hostname = hostname.hostname(config)
if current_hostname != expected_hostname:
print("Determined hostname does not match expected. Please check agent "
"log for details")
msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \
"server hostname ({1}). Aborting registration. Please check hostname, " \
"hostname -f and /etc/hosts file to confirm your " \
"hostname is setup correctly".format(current_hostname, expected_hostname)
logger.error(msg)
sys.exit(1)
# Check if there is another instance running
if os.path.isfile(ProcessHelper.pidfile) and not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
print("%s already exists, exiting" % ProcessHelper.pidfile)
sys.exit(1)
# check if ambari prefix exists
elif config.has_option('agent', 'prefix') and not os.path.isdir(os.path.abspath(config.get('agent', 'prefix'))):
msg = "Ambari prefix dir %s does not exists, can't continue" \
% config.get("agent", "prefix")
logger.error(msg)
print(msg)
sys.exit(1)
elif not config.has_option('agent', 'prefix'):
msg = "Ambari prefix dir %s not configured, can't continue"
logger.error(msg)
print(msg)
sys.exit(1)
def checkLiveServices(self, services, result):
osType = OSCheck.get_os_family()
for service in services:
svcCheckResult = {}
if isinstance(service, dict):
serviceName = service[osType]
else:
serviceName = service
service_check_live = shlex.split(self.SERVICE_STATUS_CMD)
service_check_live[1] = serviceName
svcCheckResult['name'] = serviceName
svcCheckResult['status'] = "UNKNOWN"
svcCheckResult['desc'] = ""
try:
osStat = subprocess.Popen(service_check_live, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = osStat.communicate()
if 0 != osStat.returncode:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = out
if len(out) == 0:
svcCheckResult['desc'] = err
else:
svcCheckResult['status'] = "Healthy"
except Exception, e:
svcCheckResult['status'] = "Unhealthy"
svcCheckResult['desc'] = repr(e)
result.append(svcCheckResult)
def findNearestAgentPackageVersion(projectVersion):
if projectVersion == "":
projectVersion = " "
if OSCheck.is_suse_family():
Command = ["bash", "-c", "zypper --no-gpg-checks -q search -s --match-exact ambari-agent | grep '" + projectVersion +
"' | cut -d '|' -f 4 | head -n1 | sed -e 's/-\w[^:]*//1' "]
elif OSCheck.is_ubuntu_family():
if projectVersion == " ":
Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
else:
Command = ["bash", "-c", "apt-cache -q show ambari-agent |grep 'Version\:'|cut -d ' ' -f 2|grep '" +
projectVersion + "'|tr -d '\\n'|sed -s 's/[-|~][A-Za-z0-9]*//'"]
else:
Command = ["bash", "-c", "yum -q list all ambari-agent | grep '" + projectVersion +
"' | sed -re 's/\s+/ /g' | cut -d ' ' -f 2 | head -n1 | sed -e 's/-\w[^:]*//1' "]
return execOsCommand(Command)
def allAvailablePackages(allAvailablePackages):
osType = OSCheck.get_os_family()
if OSCheck.is_suse_family():
return _lookUpZypperPackages(
["sudo", "zypper", "search", "--uninstalled-only", "--details"],
allAvailablePackages)
elif OSCheck.is_redhat_family():
return _lookUpYumPackages(
["sudo", "yum", "list", "available"],
'Available Packages',
allAvailablePackages)
elif OSCheck.is_ubuntu_family():
return _lookUpAptPackages(
LIST_AVAILABLE_PACKAGES_UBUNTU,
allAvailablePackages)
def test_get_os_version(self, mock_linux_distribution):
# 1 - Any system
mock_linux_distribution.return_value = ('', '123.45', '')
result = OSCheck.get_os_version()
self.assertEquals(result, '123.45')
# 2 - Negative case
mock_linux_distribution.return_value = ('ssss', '', 'ddddd')
try:
result = OSCheck.get_os_version()
self.fail("Should throw exception in OSCheck.get_os_version()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os version. Exiting...", str(e))
pass
def test_get_os_release_name(self, mock_linux_distribution):
# 1 - Any system
mock_linux_distribution.return_value = ('', '', 'MY_NEW_RELEASE')
result = OSCheck.get_os_release_name()
self.assertEquals(result, 'my_new_release')
# 2 - Negative case
mock_linux_distribution.return_value = ('aaaa', 'bbbb', '')
try:
result = OSCheck.get_os_release_name()
self.fail("Should throw exception in OSCheck.get_os_release_name()")
except Exception as e:
# Expected
self.assertEquals("Cannot detect os release name. Exiting...", str(e))
pass
def action_create(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = get_repo_dir()
new_content = InlineTemplate(self.resource.repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name,
base_url=self.resource.base_url, mirror_list=self.resource.mirror_list)
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
if os.path.isfile(repo_file_path):
existing_content_str = sudo.read_file(repo_file_path)
new_content_str = new_content.get_content()
if existing_content_str != new_content_str and OSCheck.is_suse_family():
# We need to reset package manager's cache when we replace base urls
# at existing repo. That is a case at least under SLES
Logger.info("Flushing package manager cache since repo file content is about to change")
checked_call(self.update_cmd, sudo=True)
if self.resource.append_to_file:
content = existing_content_str + '\n' + new_content_str
else:
content = new_content_str
else: # If repo file does not exist yet
content = new_content
File(repo_file_path,
content=content
)
def os_major_version(self):
"""
Example return value:
"6" for "Centos 6.3"
In case cannot detect --> Fail
"""
return OSCheck.get_os_major_version()
def os_family(self): """ Return values: redhat, ubuntu, suse In case cannot detect raises exception """ return OSCheck.get_os_family()
def main(argv=None):
# Same logic that was in "os_type_check.sh"
if len(sys.argv) != 2:
print "Usage: <cluster_os>"
raise Exception("Error in number of arguments. Usage: <cluster_os>")
pass
cluster_os = sys.argv[1]
current_os = OSCheck.get_os_family() + OSCheck.get_os_major_version()
# If agent/server have the same {"family","main_version"} - then ok.
print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % (
cluster_os, current_os)
if current_os == cluster_os:
sys.exit(0)
else:
raise Exception("Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host.")
def get_postgre_running_status(OS_FAMILY):
"""Return postgre running status indicator"""
if OS_FAMILY == OSConst.UBUNTU_FAMILY:
return os.path.join(get_ubuntu_pg_version(), "main")
elif OSCheck.is_redhat7():
return "active"
else:
return PG_STATUS_RUNNING_DEFAULT
def main(heartbeat_stop_callback=None):
global config
parser = OptionParser()
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
help="verbose log output",
default=False)
parser.add_option(
"-e",
"--expected-hostname",
dest="expected_hostname",
action="store",
help=
"expected hostname of current host. If hostname differs, agent will fail",
default=None)
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(),
logging_level)
global is_logger_setup
is_logger_setup = True
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(),
logging_level)
Logger.initialize_logger('resource_management',
logging_level=logging_level)
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(
config.get('agent', 'data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
#
# Iterate through the list of server hostnames and connect to the first active server
#
active_server = None
server_hostnames = hostname.server_hostnames(config)
connected = False
stopped = False
# Keep trying to connect to a server or bail out if ambari-agent was stopped
while not connected and not stopped:
for server_hostname in server_hostnames:
try:
server_ip = socket.gethostbyname(server_hostname)
server_url = config.get_api_url(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)',
server_url, server_ip)
except socket.error:
logger.warn(
"Unable to determine the IP address of the Ambari server '%s'",
server_hostname)
# Wait until MAX_RETRIES to see if server is reachable
netutil = NetUtil(heartbeat_stop_callback)
(retries, connected,
stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger)
# if connected, launch controller
if connected:
logger.info('Connected to Ambari server %s', server_hostname)
# Set the active server
active_server = server_hostname
# Launch Controller communication
controller = Controller(config, server_hostname,
heartbeat_stop_callback)
controller.start()
controller.join()
#
# If Ambari Agent connected to the server or
# Ambari Agent was stopped using stop event
# Clean up if not Windows OS
#
if connected or stopped:
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
ExitHelper().execute_cleanup()
stop_agent()
logger.info("finished")
break
pass # for server_hostname in server_hostnames
pass # while not (connected or stopped)
return active_server
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
Script.execution_command = ExecutionCommand(Script.config)
Script.module_configs = Script.execution_command.get_module_configs()
Script.cluster_settings = Script.execution_command.get_cluster_settings()
Script.stack_settings = Script.execution_command.get_stack_settings()
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
Script.repository_util = RepositoryUtil(Script.config)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if parameters is None:
return (result_code,
['There were no parameters supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if NODEMANAGER_HTTP_ADDRESS_KEY in parameters:
http_uri = parameters[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in parameters:
https_uri = parameters[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in parameters:
http_policy = parameters[YARN_HTTP_POLICY_KEY]
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme, uri)
try:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query)
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError))
return (RESULT_CODE_CRITICAL, [label])
# a map of the Ambari role to the component name
# for use with /usr/hdp/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'HIVE_METASTORE': 'hive-metastore',
'HIVE_SERVER': 'hive-server2',
'WEBHCAT_SERVER': 'hive-webhcat',
'HIVE_CLIENT': 'hive-client',
'HCAT': 'hive-client'
}
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP,
"HIVE_CLIENT")
config = Script.get_config()
if OSCheck.is_windows_family():
hive_metastore_win_service_name = "metastore"
hive_client_win_service_name = "hwi"
hive_server_win_service_name = "hiveserver2"
webhcat_server_win_service_name = "templeton"
else:
hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
hive_pid = 'hive-server.pid'
hive_metastore_pid = 'hive.pid'
hcat_pid_dir = config['configurations']['hive-env'][
'hcat_pid_dir'] #hcat_pid_dir
webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')
process_name = 'mysqld'
def __init__(self):
# OS info
self.OS_VERSION = OSCheck().get_os_major_version()
self.OS_TYPE = OSCheck.get_os_type()
self.OS_FAMILY = OSCheck.get_os_family()
def get_ntp_service():
if OSCheck.is_redhat_family():
return "ntpd"
elif OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
return "ntp"
def main(initializer_module, heartbeat_stop_callback=None):
global config
global home_dir
parser = OptionParser()
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
help="verbose log output",
default=False)
parser.add_option(
"-e",
"--expected-hostname",
dest="expected_hostname",
action="store",
help=
"expected hostname of current host. If hostname differs, agent will fail",
default=None)
parser.add_option("--home",
dest="home_dir",
action="store",
help="Home directory",
default="")
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
home_dir = options.home_dir
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(),
logging_level)
global is_logger_setup
is_logger_setup = True
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(),
logging_level)
setup_logging(alerts_logger_2,
AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
setup_logging(alerts_logger_global,
AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
setup_logging(apscheduler_logger,
AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
setup_logging(apscheduler_logger_global,
AmbariConfig.AmbariConfig.getAlertsLogFile(), logging_level)
Logger.initialize_logger('resource_management',
logging_level=logging_level)
#with Environment() as env:
# File("/abc")
# init data, once loggers are setup to see exceptions/errors of initialization.
initializer_module.init()
if home_dir != "":
# When running multiple Ambari Agents on this host for simulation, each one will use a unique home directory.
Logger.info("Agent is using Home Dir: %s" % str(home_dir))
# use the host's locale for numeric formatting
try:
locale.setlocale(locale.LC_ALL, '')
except locale.Error as ex:
logger.warning(
"Cannot set locale for ambari-agent. Please check your systemwide locale settings. Failed due to: {0}."
.format(str(ex)))
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(
config.get('agent', 'data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
update_open_files_ulimit(config)
if not config.use_system_proxy_setting():
logger.info('Agent is configured to ignore system proxy settings')
#reconfigure_urllib2_opener(ignore_system_proxy=True)
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
#
# Iterate through the list of server hostnames and connect to the first active server
#
active_server = None
server_hostnames = hostname.server_hostnames(config)
connected = False
stopped = False
# Keep trying to connect to a server or bail out if ambari-agent was stopped
while not connected and not stopped:
for server_hostname in server_hostnames:
server_url = config.get_api_url(server_hostname)
try:
server_ip = socket.gethostbyname(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)',
server_url, server_ip)
except socket.error:
logger.warn(
"Unable to determine the IP address of the Ambari server '%s'",
server_hostname)
# Wait until MAX_RETRIES to see if server is reachable
netutil = NetUtil(config, initializer_module.stop_event)
(retries, connected,
stopped) = netutil.try_to_connect(server_url, MAX_RETRIES, logger)
# if connected, launch controller
if connected:
logger.info('Connected to Ambari server %s', server_hostname)
# Set the active server
active_server = server_hostname
# Launch Controller communication
run_threads(initializer_module)
#
# If Ambari Agent connected to the server or
# Ambari Agent was stopped using stop event
# Clean up if not Windows OS
#
if connected or stopped:
ExitHelper().exit()
logger.info("finished")
break
pass # for server_hostname in server_hostnames
pass # while not (connected or stopped)
return active_server
def getOperatingSystem(self): return OSCheck.get_os_type()
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
executable_paths = None
if EXECUTABLE_SEARCH_PATHS in configurations:
executable_paths = configurations[EXECUTABLE_SEARCH_PATHS]
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
kerberos_keytab = None
if KERBEROS_KEYTAB in configurations:
kerberos_keytab = configurations[KERBEROS_KEYTAB]
kerberos_principal = None
if KERBEROS_PRINCIPAL in configurations:
kerberos_principal = configurations[KERBEROS_PRINCIPAL]
kerberos_principal = kerberos_principal.replace('_HOST', host_name)
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
label = ''
url_response = None
node_healthy = 'false'
total_time = 0
# some yarn-site structures don't have the web ui address
if uri is None:
if host_name is None:
host_name = socket.getfqdn()
uri = '{0}:{1}'.format(host_name, NODEMANAGER_DEFAULT_PORT)
if OSCheck.is_windows_family():
uri_host, uri_port = uri.split(':')
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
uri_host = resolve_address(uri_host)
uri = '{0}:{1}'.format(uri_host, uri_port)
query = "{0}://{1}/ws/v1/node/info".format(scheme,uri)
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
env = Environment.get_instance()
# curl requires an integer timeout
curl_connection_timeout = int(connection_timeout)
url_response, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal,
query, "nm_health_alert", executable_paths, False, "NodeManager Health", smokeuser,
connection_timeout=curl_connection_timeout)
json_response = json.loads(url_response)
else:
# execute the query for the JSON that includes templeton status
url_response = urllib2.urlopen(query, timeout=connection_timeout)
json_response = json.loads(url_response.read())
except urllib2.HTTPError, httpError:
label = CRITICAL_HTTP_STATUS_MESSAGE.format(str(httpError.code), query,
str(httpError), traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
def test_main(self, exithelper_exit_mock, cleanup_mock,
ping_port_init_mock, ping_port_start_mock,
data_clean_init_mock, data_clean_start_mock, parse_args_mock,
start_mock, Controller_is_alive_mock, Controller_init_mock,
try_to_connect_mock, update_log_level_mock, daemonize_mock,
perform_prestart_checks_mock, ambari_config_mock, stop_mock,
bind_signal_handlers_mock, setup_logging_mock, socket_mock):
data_clean_init_mock.return_value = None
Controller_init_mock.return_value = None
Controller_is_alive_mock.return_value = False
ping_port_init_mock.return_value = None
options = MagicMock()
parse_args_mock.return_value = (options, MagicMock)
try_to_connect_mock.return_value = (0, True, False
) # (retries, connected, stopped)
# use default unix config
ambari_config_mock.return_value = self.init_ambari_config_mock()
#testing call without command-line arguments
main.main()
self.assertTrue(setup_logging_mock.called)
self.assertTrue(perform_prestart_checks_mock.called)
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
self.assertTrue(daemonize_mock.called)
self.assertTrue(update_log_level_mock.called)
try_to_connect_mock.assert_called_once_with(ANY, main.MAX_RETRIES, ANY)
self.assertTrue(start_mock.called)
self.assertTrue(data_clean_init_mock.called)
self.assertTrue(data_clean_start_mock.called)
self.assertTrue(ping_port_init_mock.called)
self.assertTrue(ping_port_start_mock.called)
self.assertTrue(exithelper_exit_mock.called)
perform_prestart_checks_mock.reset_mock()
# Testing call with --expected-hostname parameter
options.expected_hostname = "test.hst"
main.main()
perform_prestart_checks_mock.assert_called_once_with(
options.expected_hostname)
# Test with multiple server hostnames
default_server_hostnames = hostname.cached_server_hostnames
hostname.cached_server_hostnames = ['host1', 'host2', 'host3']
def try_to_connect_impl(*args, **kwargs):
for server_hostname in hostname.cached_server_hostnames:
if (args[0].find(server_hostname) != -1):
if server_hostname == 'host1':
return 0, False, False
elif server_hostname == 'host2':
return 0, False, False
elif server_hostname == 'host3':
return 0, True, False
else:
return 0, True, False
pass
try_to_connect_mock.reset_mock()
try_to_connect_mock.side_effect = try_to_connect_impl
active_server = main.main()
self.assertEquals(active_server, 'host3')
hostname.cached_server_hostnames = default_server_hostnames
pass
def os_release_name(self):
"""
For Ubuntu 12.04:
precise
"""
return OSCheck.get_os_release_name()
# a map of the Ambari role to the component name
# for use with /usr/hdp/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'HIVE_METASTORE': 'hive-metastore',
'HIVE_SERVER': 'hive-server2',
'WEBHCAT_SERVER': 'hive-webhcat',
'HIVE_CLIENT': 'hive-client',
'HCAT': 'hive-client'
}
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP,
"HIVE_CLIENT")
config = Script.get_config()
if OSCheck.is_windows_family():
hive_metastore_win_service_name = "metastore"
hive_client_win_service_name = "hwi"
hive_server_win_service_name = "hiveserver2"
webhcat_server_win_service_name = "templeton"
else:
hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir']
hive_pid = 'hive-server.pid'
hive_metastore_pid = 'hive.pid'
hcat_pid_dir = config['configurations']['hive-env'][
'hcat_pid_dir'] #hcat_pid_dir
webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid')
process_name = 'mysqld'
def getOsFamily(self): return OSCheck.get_os_family()
def main(heartbeat_stop_callback=None):
global config
parser = OptionParser()
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
help="verbose log output",
default=False)
parser.add_option(
"-e",
"--expected-hostname",
dest="expected_hostname",
action="store",
help=
"expected hostname of current host. If hostname differs, agent will fail",
default=None)
(options, args) = parser.parse_args()
expected_hostname = options.expected_hostname
current_user = getpass.getuser()
logging_level = logging.DEBUG if options.verbose else logging.INFO
setup_logging(logger, AmbariConfig.AmbariConfig.getLogFile(),
logging_level)
setup_logging(alerts_logger, AmbariConfig.AmbariConfig.getAlertsLogFile(),
logging_level)
Logger.initialize_logger('resource_management',
logging_level=logging_level)
default_cfg = {'agent': {'prefix': '/home/ambari'}}
config.load(default_cfg)
if (len(sys.argv) > 1) and sys.argv[1] == 'stop':
stop_agent()
if (len(sys.argv) > 2) and sys.argv[1] == 'reset':
reset_agent(sys.argv)
# Check for ambari configuration file.
resolve_ambari_config()
# Add syslog hanlder based on ambari config file
add_syslog_handler(logger)
# Starting data cleanup daemon
data_cleaner = None
if config.has_option('agent', 'data_cleanup_interval') and int(
config.get('agent', 'data_cleanup_interval')) > 0:
data_cleaner = DataCleaner(config)
data_cleaner.start()
perform_prestart_checks(expected_hostname)
# Starting ping port listener
try:
#This acts as a single process machine-wide lock (albeit incomplete, since
# we still need an extra file to track the Agent PID)
ping_port_listener = PingPortListener(config)
except Exception as ex:
err_message = "Failed to start ping port listener of: " + str(ex)
logger.error(err_message)
sys.stderr.write(err_message)
sys.exit(1)
ping_port_listener.start()
update_log_level(config)
server_hostname = hostname.server_hostname(config)
server_url = config.get_api_url()
if not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
daemonize()
try:
server_ip = socket.gethostbyname(server_hostname)
logger.info('Connecting to Ambari server at %s (%s)', server_url,
server_ip)
except socket.error:
logger.warn(
"Unable to determine the IP address of the Ambari server '%s'",
server_hostname)
# Wait until server is reachable
netutil = NetUtil(heartbeat_stop_callback)
retries, connected = netutil.try_to_connect(server_url, -1, logger)
# Ambari Agent was stopped using stop event
if connected:
# Launch Controller communication
controller = Controller(config, heartbeat_stop_callback)
controller.start()
# controller.join() is not appropriate here as it blocks the signal handlers for stop etc.
while controller.is_alive():
time.sleep(0.1)
ExitHelper().exit(0)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option(
"-o",
"--out-files-logging",
dest="log_out_files",
action="store_true",
help=
"use this option to enable outputting *.out files of the service pre-start"
)
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
Logger.logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if self.command_name == "start" and not self.is_hook():
self.pre_start()
method(env)
if self.command_name == "start" and not self.is_hook():
self.post_start()
except Fail as ex:
ex.pre_raise()
raise
finally:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out()
def _collect(self):
# can be parameterized or static
# if not parameterized, this will return the static value
uri_value = self._get_configuration_value(self.uri)
if uri_value is None:
uri_value = self.host_name
logger.debug("[Alert][{0}] Setting the URI to this host since it wasn't specified".format(
self.get_name()))
# in some cases, a single property is a comma-separated list like
# host1:8080,host2:8081,host3:8083
uri_value_array = uri_value.split(',')
if len(uri_value_array) > 1:
for item in uri_value_array:
if self.host_name in item:
uri_value = item
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Extracted {1} as the host name while parsing the CSV URI {2}".format(
self.get_name(), uri_value, str(uri_value_array)))
break
host = BaseAlert.get_host_from_url(uri_value)
if host is None or host == "localhost" or host == "0.0.0.0":
host = self.host_name
try:
port = int(get_port_from_url(uri_value))
except:
if self.default_port is None:
label = 'Unable to determine port from URI {0}'.format(uri_value)
return (self.RESULT_UNKNOWN, [label])
port = self.default_port
if logger.isEnabledFor(logging.DEBUG):
logger.debug("[Alert][{0}] Checking {1} on port {2}".format(
self.get_name(), host, str(port)))
s = None
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(self.critical_timeout)
if OSCheck.is_windows_family():
# on windows 0.0.0.0 is invalid address to connect but on linux it resolved to 127.0.0.1
host = resolve_address(host)
start_time = time.time()
s.connect((host, port))
if self.socket_command is not None:
s.sendall(self.socket_command)
data = s.recv(1024)
if self.socket_command_response is not None and data != self.socket_command_response:
raise Exception("Expected response {0}, Actual response {1}".format(
self.socket_command_response, data))
end_time = time.time()
milliseconds = end_time - start_time
seconds = milliseconds / 1000.0
# not sure why this happens sometimes, but we don't always get a
# socket exception if the connect() is > than the critical threshold
if seconds >= self.critical_timeout:
return (self.RESULT_CRITICAL, ['Socket Timeout', host, port])
result = self.RESULT_OK
if seconds >= self.warning_timeout:
result = self.RESULT_WARNING
return (result, [seconds, port])
except Exception as e:
return (self.RESULT_CRITICAL, [str(e), host, port])
finally:
if s is not None:
try:
s.close()
except:
# no need to log a close failure
pass
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] == True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(
config['hostLevelParams']
['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['hostLevelParams']['agent_stack_retry_count'])
try:
available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(
config['repositoryFile']['repositories'])
except Exception as err:
available_packages_in_repos = []
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
name = self.get_package_from_available(
package['name'], available_packages_in_repos)
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
def copyNeededFiles(self):
# get the params
params = self.shared_state
# Copying the files
fileToCopy = self.getRepoFile()
target = self.getRemoteName(self.AMBARI_REPO_FILENAME)
if (os.path.exists(fileToCopy)):
self.host_log.write("==========================\n")
self.host_log.write("Copying repo file to 'tmp' folder...")
scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
target, params.bootdir, self.host_log)
retcode1 = scp.run()
self.host_log.write("\n")
# Move file to repo dir
self.host_log.write("==========================\n")
self.host_log.write("Moving file to repo dir...")
targetDir = self.getRepoDir()
command = self.getMoveRepoFileCommand(targetDir)
ssh = SSH(params.user, params.sshkey_file, self.host, command,
params.bootdir, self.host_log)
retcode2 = ssh.run()
self.host_log.write("\n")
# Change permissions on ambari.repo
self.host_log.write("==========================\n")
self.host_log.write("Changing permissions for ambari.repo...")
command = self.getRepoFileChmodCommand()
ssh = SSH(params.user, params.sshkey_file, self.host, command,
params.bootdir, self.host_log)
retcode4 = ssh.run()
self.host_log.write("\n")
# Update repo cache for ubuntu OS
if OSCheck.is_ubuntu_family():
self.host_log.write("==========================\n")
self.host_log.write("Update apt cache of repository...")
command = self.getAptUpdateCommand()
ssh = SSH(params.user, params.sshkey_file, self.host, command,
params.bootdir, self.host_log)
retcode2 = ssh.run()
self.host_log.write("\n")
retcode = max(retcode1["exitstatus"], retcode2["exitstatus"], retcode4["exitstatus"])
else:
self.host_log.write("==========================\n")
self.host_log.write("Copying required files...")
self.host_log.write("Ambari repo file not found: {0}".format(self.getRepoFile()))
retcode = -1
pass
# copy the setup script file
self.host_log.write("==========================\n")
self.host_log.write("Copying setup script file...")
fileToCopy = params.setup_agent_file
target = self.getRemoteName(self.SETUP_SCRIPT_FILENAME)
scp = SCP(params.user, params.sshkey_file, self.host, fileToCopy,
target, params.bootdir, self.host_log)
retcode3 = scp.run()
self.host_log.write("\n")
return max(retcode, retcode3["exitstatus"])
def get_postgre_running_status():
"""Return postgre running status indicator"""
if OSCheck.is_ubuntu_family():
return os.path.join(get_ubuntu_pg_version(), "main")
else:
return PG_STATUS_RUNNING_DEFAULT
def getOperatingSystemRelease(self): return OSCheck.get_os_version()
from resource_management.core.logger import Logger
from resource_management.libraries.script.script import Script
from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
from resource_management.libraries import functions
from resource_management.libraries.functions.expect import expect
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.substitute_vars import substitute_vars
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from ambari_commons.ambari_metrics_helper import select_metric_collector_hosts_from_hostnames
import status_params
from ambari_commons import OSCheck
import ConfigParser
import os
if OSCheck.is_windows_family():
from params_windows import *
else:
from params_linux import *
# server configurations
config = Script.get_config()
exec_tmp_dir = Script.get_tmp_dir()
def get_combined_memory_mb(value1, value2):
try:
part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
return str(part1 + part2) + 'm'
except:
return None
pass
def format_package_name(self, name, repo_version=None):
from resource_management.libraries.functions.default import default
"""
This function replaces ${stack_version} placeholder with actual version. If the package
version is passed from the server, use that as an absolute truth.
:param name name of the package
:param repo_version actual version of the repo currently installing
"""
stack_version_package_formatted = ""
if not repo_version:
repo_version = self.get_stack_version_before_packages_installed()
package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
# repositoryFile is the truth
# package_version should be made to the form W_X_Y_Z_nnnn
package_version = default("repositoryFile/repoVersion", None)
if package_version is not None:
package_version = package_version.replace(
'.', package_delimiter).replace('-', package_delimiter)
# TODO remove legacy checks
if package_version is None:
package_version = default("roleParams/package_version", None)
# TODO remove legacy checks
if package_version is None:
package_version = default("hostLevelParams/package_version", None)
# The cluster effective version comes down when the version is known after the initial
# install. In that case we should not be guessing which version when invoking INSTALL, but
# use the supplied version to build the package_version
effective_version = default("commandParams/version", None)
role_command = default("roleCommand", None)
if (package_version is None or '*' in package_version) \
and effective_version is not None and 'INSTALL' == role_command:
package_version = effective_version.replace(
'.', package_delimiter).replace('-', package_delimiter)
Logger.info(
"Version {0} was provided as effective cluster version. Using package version {1}"
.format(effective_version, package_version))
if package_version:
stack_version_package_formatted = package_version
if OSCheck.is_ubuntu_family():
stack_version_package_formatted = package_version.replace(
'_', package_delimiter)
# Wildcards cause a lot of troubles with installing packages, if the version contains wildcards we try to specify it.
if not package_version or '*' in package_version:
stack_version_package_formatted = repo_version.replace(
'.', package_delimiter).replace(
'-', package_delimiter
) if STACK_VERSION_PLACEHOLDER in name else name
package_name = name.replace(STACK_VERSION_PLACEHOLDER,
stack_version_package_formatted)
return package_name
