def phoenix_service(action = 'start'): # 'start', 'stop', 'status'
# Note: params/status_params should already be imported before calling phoenix_service()
pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
if action == "status":
check_process_status(pid_file)
else:
env = {'JAVA_HOME': format("{java64_home}"), 'HBASE_CONF_DIR': format("{hbase_conf_dir}")}
daemon_cmd = format("{phx_daemon_script} {action}")
if action == 'start':
Execute(daemon_cmd,
user=format("{hbase_user}"),
environment=env)
elif action == 'stop':
Execute(daemon_cmd,
timeout = 30,
on_timeout = format("! ( {no_op_test} ) || {sudo} -H -E kill -9 `cat {pid_file}`"),
user=format("{hbase_user}"),
environment=env
)
File(pid_file,
action = "delete"
)
def phoenix_service(action='start'): # 'start', 'stop', 'status'
# Note: params/status_params should already be imported before calling phoenix_service()
pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid")
no_op_test = format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
)
if action == "status":
check_process_status(pid_file)
else:
env = {
'JAVA_HOME': format("{java64_home}"),
'HBASE_CONF_DIR': format("{hbase_conf_dir}")
}
daemon_cmd = format("{phx_daemon_script} {action}")
if action == 'start':
Execute(daemon_cmd, user=format("{hbase_user}"), environment=env)
elif action == 'stop':
Execute(
daemon_cmd,
timeout=30,
on_timeout=format(
"! ( {no_op_test} ) || {sudo} -H -E kill -9 `cat {pid_file}`"
),
user=format("{hbase_user}"),
environment=env)
File(pid_file, action="delete")
def hbase_thrift_server(action='start'): # 'start', 'stop', 'status'
pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid")
no_op_test = format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
)
if action == "status":
check_process_status(pid_file)
else:
env = {
'JAVA_HOME': format("{java64_home}"),
'HBASE_CONF_DIR': format("{hbase_conf_dir}")
}
if action == 'start':
Execute(format(
"{thrift_daemon_script} {action} thrift -p 9090 --infoport 9080"
),
user=format("{hbase_user}"),
environment=env)
elif action == 'stop':
Execute(
format("{thrift_daemon_script} {action} thrift"),
timeout=30,
on_timeout=format(
"! ( {no_op_test} ) || {sudo} -H -E kill -9 `cat {pid_file}`"
),
user=format("{hbase_user}"),
environment=env)
File(pid_file, action="delete")
def status(self, env):
# This custom log was added because the normal ambari logging does not work here
# import logging
# logger = logging.getLogger('mongo')
# logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
# fh = logging.FileHandler('/var/log/ambari-agent/mongo.log')
# fh.setLevel(logging.DEBUG)
# logger.addHandler(fh)
self.log("Initiating mongo status...")
self.configure(env)
my_hostname = self.my_hostname
self.log('My hostname: ' + my_hostname)
self.log("Checking mongo instances status...")
# the parameter withThisHostInstancesOnly is being used because when the agent executes this commmand, we do not
# have the ambari hosts lists available. The ambari config "clusterHostInfo" is not available here
cluster_status = self.getClusterStatus(self.getClusterData(withThisHostInstancesOnly=True))
self.log('Shards to process: ' + str(len(cluster_status)))
for shard in cluster_status:
self.log('Processing shard: ' + shard[0])
self.log('Nodes to process: ' + str(len(shard[2])))
for node in shard[2]:
self.log('Processing node: ' + node.host_name + ":" + node.db_port)
self.log('The node is started: ' + str(node.is_started))
self.log('Pid file :' + node.pid_file_name)
if node.host_name == my_hostname:
self.log('Checking process id ...')
check_process_status(node.pid_file_name)
def phoenix_service(action = 'start'): # 'start', 'stop', 'status'
# Note: params/status_params should already be imported before calling phoenix_service()
pid_file = format("{pid_dir}/phoenix-{hbase_user}-server.pid")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1")
if action == "status":
check_process_status(pid_file)
else:
env = {'JAVA_HOME': format("{java64_home}"), 'HBASE_CONF_DIR': format("{hbase_conf_dir}")}
daemon_cmd = format("{phx_daemon_script} {action}")
if action == 'start':
Execute(daemon_cmd,
user=format("{hbase_user}"),
environment=env)
elif action == 'stop':
Execute(daemon_cmd,
timeout = 30,
on_timeout = format("! ( {no_op_test} ) || {sudo} -H -E kill -9 `cat {pid_file}`"),
user=format("{hbase_user}"),
environment=env
)
try:
File(pid_file, action = "delete")
except OSError as exc:
# OSError: [Errno 2] No such file or directory
if exc.errno == errno.ENOENT:
Logger.info("Did not remove '{0}' as it did not exist".format(pid_file))
else:
raise
def is_datanode_process_running():
import params
try:
check_process_status(params.datanode_pid_file)
return True
except ComponentIsNotRunning:
return False
def is_region_server_process_running():
try:
pid_file = format("{pid_dir}/hbase-{hbase_user}-regionserver.pid")
check_process_status(pid_file)
return True
except ComponentIsNotRunning:
return False
def status(self, env):
import status_params
from resource_management.libraries.functions import check_process_status
env.set_params(status_params)
# Recursively check all existing gmetad pid files
check_process_status(status_params.hive_metastore_pid)
def status(self, env):
import status_params
from resource_management.libraries.functions import check_process_status
env.set_params(status_params)
pid_file = format("{hive_pid_dir}/{hive_metastore_pid}")
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
def status(self, env):
import status_params
env.set_params(status_params)
Execute(format(
"mv {yarn_historyserver_pid_file_old} {yarn_historyserver_pid_file}"
),
only_if=format("test -e {yarn_historyserver_pid_file_old}",
user=status_params.yarn_user))
check_process_status(status_params.yarn_historyserver_pid_file)
def is_flume_process_live(pid_file):
"""
Gets whether the flume agent represented by the specified file is running.
:param pid_file: the PID file of the agent to check
:return: True if the agent is running, False otherwise
"""
live = False
try:
check_process_status(pid_file)
live = True
except ComponentIsNotRunning:
pass
return live
def is_flume_process_live(pid_file):
"""
Gets whether the flume agent represented by the specified file is running.
:param pid_file: the PID file of the agent to check
:return: True if the agent is running, False otherwise
"""
live = False
try:
check_process_status(pid_file)
live = True
except ComponentIsNotRunning:
pass
return live
def status(self, env): import status_params env.set_params(status_params) check_process_status(status_params.server_pid_file)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.hive_metastore_pid)
def status(self, env):
import status_params
env.set_params(status_params)
component = self.component
pid_file = format("{pid_dir}/accumulo-{accumulo_user}-{component}.pid")
check_process_status(pid_file)
def status(self, env):
import params_status
env.set_params(params_status)
check_process_status(params_status.drill_run_dir + '/drillbit.pid')
def status(self, env):
import status_params
env.set_params(status_params)
component = self.component
pid_file = format("{pid_dir}/accumulo-{accumulo_user}-{component}.pid")
check_process_status(pid_file)
def service(action=None, name=None, user=None, options="", create_pid_dir=False,
create_log_dir=False):
"""
:param action: Either "start" or "stop"
:param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc"
:param user: User to run the command as
:param options: Additional options to pass to command as a string
:param create_pid_dir: Create PID directory
:param create_log_dir: Crate log file directory
"""
import params
options = options if options else ""
pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
hadoop_env_exports = {
'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir
}
log_dir = format("{hdfs_log_dir_prefix}/{user}")
# NFS GATEWAY is always started by root using jsvc due to rpcbind bugs
# on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542
if name == "nfs3" :
pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid")
custom_export = {
'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user,
'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir,
'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir
}
hadoop_env_exports.update(custom_export)
process_id_exists_command = as_sudo(["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# on STOP directories shouldn't be created
# since during stop still old dirs are used (which were created during previous start)
if action != "stop":
if name == "nfs3":
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.root_user,
group=params.root_group
)
else:
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group
)
if create_pid_dir:
Directory(pid_dir,
owner=user,
recursive=True)
if create_log_dir:
if name == "nfs3":
Directory(log_dir,
mode=0775,
owner=params.root_user,
group=params.user_group)
else:
Directory(log_dir,
owner=user,
recursive=True)
if params.security_enabled and name == "datanode":
## The directory where pid files are stored in the secure data environment.
hadoop_secure_dn_pid_dir = format("{hadoop_pid_dir_prefix}/{hdfs_user}")
hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
# At Champlain stack and further, we may start datanode as a non-root even in secure cluster
if not (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) or params.secure_dn_ports_are_in_use:
user = "******"
pid_file = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid")
if action == 'stop' and (params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0) and \
os.path.isfile(hadoop_secure_dn_pid_file):
# We need special handling for this case to handle the situation
# when we configure non-root secure DN and then restart it
# to handle new configs. Otherwise we will not be able to stop
# a running instance
user = "******"
try:
check_process_status(hadoop_secure_dn_pid_file)
custom_export = {
'HADOOP_SECURE_DN_USER': params.hdfs_user
}
hadoop_env_exports.update(custom_export)
except ComponentIsNotRunning:
pass
hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh")
if user == "root":
cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir, action, name]
if options:
cmd += [options, ]
daemon_cmd = as_sudo(cmd)
else:
cmd = format("{ulimit_cmd} {hadoop_daemon} --config {hadoop_conf_dir} {action} {name}")
if options:
cmd += " " + options
daemon_cmd = as_user(cmd, user)
if action == "start":
# remove pid file from dead process
File(pid_file, action="delete", not_if=process_id_exists_command)
Execute(daemon_cmd, not_if=process_id_exists_command, environment=hadoop_env_exports)
elif action == "stop":
Execute(daemon_cmd, only_if=process_id_exists_command, environment=hadoop_env_exports)
File(pid_file, action="delete")
def stopped(pid_file):
try:
check_process_status(pid_file)
return False
except ComponentIsNotRunning:
return True
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.yarn_historyserver_pid_file)
def status(self, env):
check_process_status('/var/run/nginx/nginx.pid')
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.pid_ui)
def status(self, env):
import params
env.set_params(params)
check_process_status(params.drill_pid_file)
def status(self, env):
import status_params
env.set_params(status_params)
check_process_status(status_params.webhcat_pid_file)
def status(self, env): import params env.set_params(params) check_process_status(params.drill_pid_file)
def status(self, env):
import status_params
env.set_params(status_params)
for pid_file in self.get_pid_files():
check_process_status(pid_file)
def status(self, env):
import status_params
env.set_params(status_params)
# Recursively check all existing gmetad pid files
check_process_status(status_params.hive_metastore_pid)
def service(action=None,
name=None,
user=None,
options="",
create_pid_dir=False,
create_log_dir=False):
"""
:param action: Either "start" or "stop"
:param name: Component name, e.g., "namenode", "datanode", "secondarynamenode", "zkfc"
:param user: User to run the command as
:param options: Additional options to pass to command as a string
:param create_pid_dir: Create PID directory
:param create_log_dir: Crate log file directory
"""
import params
options = options if options else ""
pid_dir = format("{hadoop_pid_dir_prefix}/{user}")
pid_file = format("{pid_dir}/hadoop-{user}-{name}.pid")
hadoop_env_exports = {'HADOOP_LIBEXEC_DIR': params.hadoop_libexec_dir}
log_dir = format("{hdfs_log_dir_prefix}/{user}")
# NFS GATEWAY is always started by root using jsvc due to rpcbind bugs
# on Linux such as CentOS6.2. https://bugzilla.redhat.com/show_bug.cgi?id=731542
if name == "nfs3":
pid_file = format("{pid_dir}/hadoop_privileged_nfs3.pid")
custom_export = {
'HADOOP_PRIVILEGED_NFS_USER': params.hdfs_user,
'HADOOP_PRIVILEGED_NFS_PID_DIR': pid_dir,
'HADOOP_PRIVILEGED_NFS_LOG_DIR': log_dir
}
hadoop_env_exports.update(custom_export)
process_id_exists_command = as_sudo(
["test", "-f", pid_file]) + " && " + as_sudo(["pgrep", "-F", pid_file])
# on STOP directories shouldn't be created
# since during stop still old dirs are used (which were created during previous start)
if action != "stop":
if name == "nfs3":
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.root_user,
group=params.root_group)
else:
Directory(params.hadoop_pid_dir_prefix,
mode=0755,
owner=params.hdfs_user,
group=params.user_group)
if create_pid_dir:
Directory(pid_dir,
owner=user,
group=params.user_group,
create_parents=True)
if create_log_dir:
if name == "nfs3":
Directory(log_dir,
mode=0775,
owner=params.root_user,
group=params.user_group)
else:
Directory(log_dir,
owner=user,
group=params.user_group,
create_parents=True)
if params.security_enabled and name == "datanode":
## The directory where pid files are stored in the secure data environment.
hadoop_secure_dn_pid_dir = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}")
hadoop_secure_dn_pid_file = format(
"{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
# At datanode_non_root stack version and further, we may start datanode as a non-root even in secure cluster
if not (params.stack_version_formatted and check_stack_feature(
StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)
) or params.secure_dn_ports_are_in_use:
user = "******"
pid_file = format(
"{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid"
)
if action == 'stop' and (params.stack_version_formatted and check_stack_feature(StackFeature.DATANODE_NON_ROOT, params.stack_version_formatted)) and \
os.path.isfile(hadoop_secure_dn_pid_file):
# We need special handling for this case to handle the situation
# when we configure non-root secure DN and then restart it
# to handle new configs. Otherwise we will not be able to stop
# a running instance
user = "******"
try:
check_process_status(hadoop_secure_dn_pid_file)
custom_export = {'HADOOP_SECURE_DN_USER': params.hdfs_user}
hadoop_env_exports.update(custom_export)
except ComponentIsNotRunning:
pass
hdfs_bin = format("{hadoop_bin}/hdfs")
if user == "root":
cmd = [
hdfs_bin, "--config", params.hadoop_conf_dir, "--daemon", action,
name
]
if options:
cmd += [
options,
]
daemon_cmd = as_sudo(cmd)
else:
cmd = format(
"{ulimit_cmd} {hdfs_bin} --config {hadoop_conf_dir} --daemon {action} {name}"
)
if options:
cmd += " " + options
daemon_cmd = as_user(cmd, user)
if action == "start":
# remove pid file from dead process
File(pid_file, action="delete", not_if=process_id_exists_command)
try:
Execute(daemon_cmd,
not_if=process_id_exists_command,
environment=hadoop_env_exports)
except:
show_logs(log_dir, user)
raise
elif action == "stop":
try:
Execute(daemon_cmd,
only_if=process_id_exists_command,
environment=hadoop_env_exports)
except:
show_logs(log_dir, user)
raise
wait_process_stopped(pid_file)
File(pid_file, action="delete")
def status(self, env):
import params
check_process_status(params.master_pid_file)
def status(self, env): import status_params env.set_params(status_params) check_process_status(status_params.pid_drpc)
def status(self, env):
import status_params
env.set_params(status_params)
pid_file = self.get_pid_files()[0]
check_process_status(pid_file)
