def launch_python_subprocess(self, command, tmpout, tmperr):
"""
Creates subprocess with given parameters. This functionality was moved to separate method
to make possible unit testing
"""
close_fds = None if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else True
command_env = dict(os.environ)
if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
command_env["PYTHONPATH"] = os.pathsep.join(sys.path)
for k, v in command_env.iteritems():
command_env[k] = str(v)
return subprocess.Popen(command,
stdout=tmpout,
stderr=tmperr, close_fds=close_fds, env=command_env, preexec_fn=self.preexec_fn)
def launch_python_subprocess(self, command, tmpout, tmperr):
"""
Creates subprocess with given parameters. This functionality was moved to separate method
to make possible unit testing
"""
close_fds = None if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else True
command_env = dict(os.environ)
if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
command_env["PYTHONPATH"] = os.pathsep.join(sys.path)
for k, v in command_env.iteritems():
command_env[k] = str(v)
return subprocess.Popen(command,
stdout=tmpout,
stderr=tmperr, close_fds=close_fds, env=command_env, preexec_fn=self.preexec_fn)
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
if parameters is None:
return (RESULT_CODE_UNKNOWN,
['There were no parameters supplied to the script.'])
if not OOZIE_URL_KEY in parameters:
return (RESULT_CODE_UNKNOWN,
['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family(
) == OSConst.WINSRV_FAMILY else '0.0.0.0'
oozie_url = parameters[OOZIE_URL_KEY]
oozie_url = oozie_url.replace(
urlparse(oozie_url).hostname, localhost_address)
try:
command, env = get_check_command(oozie_url, host_name, parameters)
# execute the command
Execute(command, environment=env)
return (RESULT_CODE_OK,
["Successful connection to {0}".format(oozie_url)])
except KerberosPropertiesNotFound, ex:
return (RESULT_CODE_UNKNOWN, [str(ex)])
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
if configurations is None:
return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.'])
if not OOZIE_URL_KEY in configurations:
return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0'
oozie_url = configurations[OOZIE_URL_KEY]
oozie_url = oozie_url.replace(urlparse(oozie_url).hostname,localhost_address)
try:
command, env = get_check_command(oozie_url, host_name, configurations)
# execute the command
Execute(command, environment=env)
return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)])
except KerberosPropertiesNotFound, ex:
return (RESULT_CODE_UNKNOWN, [str(ex)])
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
if configurations is None:
return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.'])
if not OOZIE_URL_KEY in configurations:
return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0'
https_port = None
# try to get https port form oozie-env content
if OOZIE_ENV_CONTENT in configurations:
for line in configurations[OOZIE_ENV_CONTENT].splitlines():
result = re.match(OOZIE_ENV_HTTPS_RE, line)
if result is not None:
https_port = result.group(1)
# or from oozie-site.xml
if https_port is None and OOZIE_HTTPS_PORT in configurations:
https_port = configurations[OOZIE_HTTPS_PORT]
oozie_url = configurations[OOZIE_URL_KEY]
# construct proper url for https
if https_port is not None:
parsed_url = urlparse(oozie_url)
oozie_url = oozie_url.replace(parsed_url.scheme, "https")
if parsed_url.port is None:
oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)]))
else:
oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port))
# https will not work with localhost address, we need put fqdn
if https_port is None:
oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, localhost_address)
try:
command, env, oozie_user = get_check_command(oozie_url, host_name, configurations)
# execute the command
Execute(command,
environment=env,
user=oozie_user,
)
return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)])
except KerberosPropertiesNotFound, ex:
return (RESULT_CODE_UNKNOWN, [str(ex)])
def getRepoDir(self):
if OSCheck.is_redhat_family():
return "/etc/yum.repos.d"
elif OSCheck.is_suse_family():
return "/etc/zypp/repos.d"
elif OSCheck.is_ubuntu_family():
return "/etc/apt/sources.list.d"
else:
raise Exception("Unsupported OS family '{0}'".format(
OSCheck.get_os_family()))
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
if configurations is None:
return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.'])
if not OOZIE_URL_KEY in configurations:
return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0'
https_port = None
# try to get https port form oozie-env content
if OOZIE_ENV_CONTENT in configurations:
for line in configurations[OOZIE_ENV_CONTENT].splitlines():
result = re.match(OOZIE_ENV_HTTPS_RE, line)
if result is not None:
https_port = result.group(1)
# or from oozie-site.xml
if https_port is None and OOZIE_HTTPS_PORT in configurations:
https_port = configurations[OOZIE_HTTPS_PORT]
oozie_url = configurations[OOZIE_URL_KEY]
# construct proper url for https
if https_port is not None:
parsed_url = urlparse(oozie_url)
oozie_url = oozie_url.replace(parsed_url.scheme, "https")
if parsed_url.port is None:
oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)]))
else:
oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port))
# https will not work with localhost address, we need put fqdn
if https_port is None:
oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, localhost_address)
try:
command, env, smokeuser = get_check_command(oozie_url, host_name, configurations, parameters)
# execute the command
Execute(command, environment=env, user=smokeuser)
return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)])
except KerberosPropertiesNotFound, ex:
return (RESULT_CODE_UNKNOWN, [str(ex)])
def bind_signal_handlers(agentPid):
global _handler
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGUSR1, debug)
_handler = HeartbeatStopHandlersLinux()
else:
_handler = HeartbeatStopHandlersWindows()
return _handler
def bind_signal_handlers(agentPid):
global _handler
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGUSR1, debug)
_handler = HeartbeatStopHandlersLinux()
else:
_handler = HeartbeatStopHandlersWindows()
return _handler
def get_clearcache_cmd(self):
if OSCheck.is_redhat_family():
Logger.info("Clear repository cache for the RedHat OS family");
return ("/usr/bin/yum", "clean", "all")
elif OSCheck.is_suse_family():
Logger.info("Clear repository cache for the SUSE OS family");
return ('/usr/bin/zypper', 'refresh')
elif OSCheck.is_ubuntu_family():
Logger.info("Clear repository cache for the Ubuntu OS family");
return ('/usr/bin/apt-get', 'update')
else:
raise Exception("Unsupported OS family: '{0}' ".format(OSCheck.get_os_family()))
def bind_signal_handlers(agentPid, stop_event):
global _handler
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
bind_debug_signal_handlers()
_handler = stop_event
else:
_handler = stop_event
return _handler
def bind_signal_handlers(agentPid):
global _handler
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGUSR2, remote_debug) # Interrupt running process, and provide a python prompt for it
try:
import faulthandler # This is not default module, has to be installed separately
faulthandler.enable(file=sys.stderr, all_threads=True)
faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True, chain=False)
sys.stderr.write("Registered faulthandler\n")
except ImportError:
pass # Module is not included into python distribution
_handler = HeartbeatStopHandlersLinux()
else:
_handler = HeartbeatStopHandlersWindows()
return _handler
def bind_signal_handlers(agentPid):
global _handler
if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY:
if os.getpid() == agentPid:
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
try:
import faulthandler # This is not default module, has to be installed separately
faulthandler.enable(file=sys.stderr, all_threads=True)
faulthandler.register(signal.SIGUSR1,
file=sys.stderr,
all_threads=True,
chain=False)
sys.stderr.write("Registered faulthandler\n")
except ImportError:
pass # Module is not included into python distribution
_handler = HeartbeatStopHandlersLinux()
else:
_handler = HeartbeatStopHandlersWindows()
return _handler
def get_provider(resource_type):
"""
Looking for {resource_type} provider implementation for current os and returning ready to use instance
:param resource_type existing provider type, case sensitive
:type resource_type str
:raise Fail
:return Provider instance
"""
os_family = OSCheck.get_os_family()
providers = [PROVIDERS, LIBRARY_PROVIDERS]
os_family_provider = None
class_path = None
for provider in providers:
if os_family in provider:
os_family_provider = provider[os_family]
else:
# take care of os extensions
for family in provider:
if OSCheck.is_in_family(os_family, family):
os_family_provider = provider[family]
if os_family_provider and resource_type in os_family_provider:
class_path = os_family_provider[resource_type]
break
if resource_type in provider["default"]:
class_path = provider["default"][resource_type]
break
try:
mod_path, class_name = class_path.rsplit('.', 1)
except ValueError:
raise Fail("Unable to find provider for %s as %s" %
(resource_type, class_path))
mod = __import__(mod_path, {}, {}, [class_name])
return getattr(mod, class_name)(None)
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
if parameters is None:
return (RESULT_CODE_UNKNOWN, ['There were no parameters supplied to the script.'])
if not OOZIE_URL_KEY in parameters:
return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0'
oozie_url = parameters[OOZIE_URL_KEY]
oozie_url = oozie_url.replace(urlparse(oozie_url).hostname,localhost_address)
security_enabled = False
if SECURITY_ENABLED in parameters:
security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'
command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status")
try:
# kinit if security is enabled so that oozie-env.sh can make the web request
kerberos_env = None
if security_enabled:
if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
oozie_keytab = parameters[OOZIE_KEYTAB]
oozie_principal = parameters[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
return (RESULT_CODE_UNKNOWN, ['The Oozie keytab and principal are required parameters when security is enabled.'])
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, sep, getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
klist_path_local = get_klist_path()
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command)
if return_code != 0:
kinit_path_local = get_kinit_path()
kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")
# kinit
Execute(kinit_command, environment=kerberos_env)
# execute the command
Execute(command, environment=kerberos_env)
return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)])
except Exception, ex:
return (RESULT_CODE_CRITICAL, [str(ex)])
import tempfile from ambari_commons.exceptions import FatalException from ambari_commons.os_check import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons.os_utils import run_os_command, search_file, set_file_permissions from ambari_commons.logging_utils import get_debug_mode, print_info_msg, print_warning_msg, print_error_msg, \ set_debug_mode from ambari_server.properties import Properties from ambari_server.userInput import get_validated_string_input from ambari_server.utils import compare_versions, locate_file OS_VERSION = OSCheck().get_os_major_version() OS_TYPE = OSCheck.get_os_type() OS_FAMILY = OSCheck.get_os_family() PID_NAME = "ambari-server.pid" # Non-root user setup commands NR_USER_PROPERTY = "ambari-server.user" BLIND_PASSWORD = "******" # Common messages PRESS_ENTER_MSG = "Press <enter> to continue." OS_FAMILY_PROPERTY = "server.os_family" OS_TYPE_PROPERTY = "server.os_type" BOOTSTRAP_DIR_PROPERTY = "bootstrap.dir"
def osdisks():
if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY:
return Hardware._osdisks_win()
else:
return Hardware._osdisks_linux()
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
if parameters is None:
return (RESULT_CODE_UNKNOWN,
['There were no parameters supplied to the script.'])
if not OOZIE_URL_KEY in parameters:
return (RESULT_CODE_UNKNOWN,
['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family(
) == OSConst.WINSRV_FAMILY else '0.0.0.0'
oozie_url = parameters[OOZIE_URL_KEY]
oozie_url = oozie_url.replace(
urlparse(oozie_url).hostname, localhost_address)
security_enabled = False
if SECURITY_ENABLED in parameters:
security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'
command = format(
"source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status"
)
try:
# kinit if security is enabled so that oozie-env.sh can make the web request
if security_enabled:
if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
oozie_keytab = parameters[OOZIE_KEYTAB]
oozie_principal = parameters[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
return (RESULT_CODE_UNKNOWN, [
'The Oozie keytab and principal are required parameters when security is enabled.'
])
kinit_path_local = get_kinit_path(
["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
kinit_command = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal}; ")
# kinit
Execute(kinit_command)
# execute the command
Execute(command)
return (RESULT_CODE_OK,
["Successful connection to {0}".format(oozie_url)])
except Exception, ex:
return (RESULT_CODE_CRITICAL, [str(ex)])
def execute(parameters=None, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
parameters (dictionary): a mapping of parameter key to value
host_name (string): the name of this host where the alert is running
"""
if parameters is None:
return (RESULT_CODE_UNKNOWN,
['There were no parameters supplied to the script.'])
if not OOZIE_URL_KEY in parameters:
return (RESULT_CODE_UNKNOWN,
['The Oozie URL is a required parameter.'])
# use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all
# interfaces, which doesn't work on Windows
localhost_address = 'localhost' if OSCheck.get_os_family(
) == OSConst.WINSRV_FAMILY else '0.0.0.0'
oozie_url = parameters[OOZIE_URL_KEY]
oozie_url = oozie_url.replace(
urlparse(oozie_url).hostname, localhost_address)
security_enabled = False
if SECURITY_ENABLED in parameters:
security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE'
command = format(
"source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status"
)
try:
# kinit if security is enabled so that oozie-env.sh can make the web request
kerberos_env = None
if security_enabled:
if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters:
oozie_keytab = parameters[OOZIE_KEYTAB]
oozie_principal = parameters[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
return (RESULT_CODE_UNKNOWN, [
'The Oozie keytab and principal are required parameters when security is enabled.'
])
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(
env.tmp_dir, sep, getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
klist_path_local = get_klist_path()
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command)
if return_code != 0:
kinit_path_local = get_kinit_path()
kinit_command = format(
"{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; "
)
# kinit
Execute(kinit_command, environment=kerberos_env)
# execute the command
Execute(command, environment=kerberos_env)
return (RESULT_CODE_OK,
["Successful connection to {0}".format(oozie_url)])
except Exception, ex:
return (RESULT_CODE_CRITICAL, [str(ex)])
import tempfile from ambari_commons.exceptions import FatalException from ambari_commons.os_check import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons.os_utils import run_os_command, search_file, set_file_permissions from ambari_commons.logging_utils import get_debug_mode, print_info_msg, print_warning_msg, print_error_msg, \ set_debug_mode from ambari_server.properties import Properties from ambari_server.userInput import get_validated_string_input from ambari_server.utils import compare_versions, locate_file OS_VERSION = OSCheck().get_os_major_version() OS_TYPE = OSCheck.get_os_type() OS_FAMILY = OSCheck.get_os_family() PID_NAME = "ambari-server.pid" # Non-root user setup commands NR_USER_PROPERTY = "ambari-server.user" BLIND_PASSWORD = "******" # Common messages PRESS_ENTER_MSG = "Press <enter> to continue." OS_FAMILY_PROPERTY = "server.os_family" OS_TYPE_PROPERTY = "server.os_type" BOOTSTRAP_DIR_PROPERTY = "bootstrap.dir"
