def top(num_processes=5, interval=3):
'''
Return a list of top CPU consuming processes during the interval.
num_processes = return the top N CPU consuming processes
interval = the number of seconds to sample CPU usage over
CLI Examples:
.. code-block:: bash
salt '*' ps.top
salt '*' ps.top 5 10
'''
result = []
start_usage = {}
for pid in psutil.pids():
try:
process = psutil.Process(pid)
user, system = process.cpu_times()
except ValueError:
user, system, _, _ = process.cpu_times()
except psutil.NoSuchProcess:
continue
start_usage[process] = user + system
time.sleep(interval)
usage = set()
for process, start in six.iteritems(start_usage):
try:
user, system = process.cpu_times()
except ValueError:
user, system, _, _ = process.cpu_times()
except psutil.NoSuchProcess:
continue
now = user + system
diff = now - start
usage.add((diff, process))
for idx, (diff, process) in enumerate(reversed(sorted(usage))):
if num_processes and idx >= num_processes:
break
if len(_get_proc_cmdline(process)) == 0:
cmdline = _get_proc_name(process)
else:
cmdline = _get_proc_cmdline(process)
info = {'cmd': cmdline,
'user': _get_proc_username(process),
'status': _get_proc_status(process),
'pid': _get_proc_pid(process),
'create_time': _get_proc_create_time(process),
'cpu': {},
'mem': {},
}
for key, value in six.iteritems(process.cpu_times()._asdict()):
info['cpu'][key] = value
for key, value in six.iteritems(process.memory_info()._asdict()):
info['mem'][key] = value
result.append(info)
return result
def kill_pid(pid, signal=15):
'''
Kill a process by PID.
.. code-block:: bash
salt 'minion' ps.kill_pid pid [signal=signal_number]
pid
PID of process to kill.
signal
Signal to send to the process. See manpage entry for kill
for possible values. Default: 15 (SIGTERM).
**Example:**
Send SIGKILL to process with PID 2000:
.. code-block:: bash
salt 'minion' ps.kill_pid 2000 signal=9
'''
try:
psutil.Process(pid).send_signal(signal)
return True
except psutil.NoSuchProcess:
return False
def is_pid_healthy(pid):
'''
This is a health check that will confirm the PID is running
and executed by salt.
If pusutil is available:
* all architectures are checked
if psutil is not available:
* Linux/Solaris/etc: archs with `/proc/cmdline` available are checked
* AIX/Windows: assume PID is healhty and return True
'''
if HAS_PSUTIL:
try:
proc = psutil.Process(pid)
except psutil.NoSuchProcess:
log.warning("PID %s is no longer running.", pid)
return False
return any(['salt' in cmd for cmd in proc.cmdline()])
if salt.utils.platform.is_aix() or salt.utils.platform.is_windows():
return True
if not salt.utils.process.os_is_running(pid):
log.warning("PID %s is no longer running.", pid)
return False
cmdline_file = os.path.join('proc', str(pid), 'cmdline')
try:
with salt.utils.files.fopen(cmdline_file, 'rb') as fp_:
return b'salt' in fp_.read()
except (OSError, IOError) as err:
log.error("There was a problem reading proc file: %s", err)
return False
def signal_job(jid, sig):
'''
Sends a signal to the named salt job's process
CLI Example:
.. code-block:: bash
salt '*' saltutil.signal_job <job id> 15
'''
for data in running():
if data['jid'] == jid:
try:
for proc in psutil.Process(pid=data['pid']).children(
recursive=True):
proc.send_signal(sig)
os.kill(int(data['pid']), sig)
return 'Signal {0} sent to job {1} at pid {2}'.format(
int(sig), jid, data['pid'])
except OSError:
path = os.path.join(__opts__['cachedir'], 'proc', str(jid))
if os.path.isfile(path):
os.remove(path)
return ('Job {0} was not running and job data has been '
' cleaned up').format(jid)
return ''
def test_top_zombie_process(self):
# Get 3 pids that are currently running on the system
pids = psutil.pids()[:3]
# Get a process instance for each of the pids
processes = [psutil.Process(pid) for pid in pids]
# Patch the middle process to raise ZombieProcess when .cpu_times is called
def raise_exception():
raise psutil.ZombieProcess(processes[1].pid)
processes[1].cpu_times = raise_exception
# Make sure psutil.pids only returns the above 3 pids
with patch("salt.utils.psutil_compat.pids", return_value=pids):
# Make sure we use our process list from above
with patch("salt.utils.psutil_compat.Process",
side_effect=processes):
result = ps.top(num_processes=1, interval=0)
assert len(result) == 1
def proc_info(pid, attrs=None):
'''
Return a dictionary of information for a process id (PID).
CLI Example:
.. code-block:: bash
salt '*' ps.proc_info 2322
salt '*' ps.proc_info 2322 attrs='["pid", "name"]'
pid
PID of process to query.
attrs
Optional list of desired process attributes. The list of possible
attributes can be found here:
http://pythonhosted.org/psutil/#psutil.Process
'''
try:
proc = psutil.Process(pid)
return proc.as_dict(attrs)
except (psutil.NoSuchProcess, psutil.AccessDenied, AttributeError) as exc:
raise CommandExecutionError(exc)
def top(num_processes=5, interval=3):
"""
Return a list of top CPU consuming processes during the interval.
num_processes = return the top N CPU consuming processes
interval = the number of seconds to sample CPU usage over
CLI Examples:
.. code-block:: bash
salt '*' ps.top
salt '*' ps.top 5 10
"""
result = []
start_usage = {}
for pid in psutil.pids():
try:
process = psutil.Process(pid)
except psutil.NoSuchProcess:
continue
else:
try:
user, system = process.cpu_times()[:2]
except psutil.ZombieProcess:
user = system = 0.0
start_usage[process] = user + system
time.sleep(interval)
usage = set()
for process, start in six.iteritems(start_usage):
try:
user, system = process.cpu_times()[:2]
except psutil.NoSuchProcess:
continue
now = user + system
diff = now - start
usage.add((diff, process))
for diff, process in sorted(usage, key=lambda x: x[0], reverse=True):
info = {
"cmd": _get_proc_cmdline(process) or _get_proc_name(process),
"user": _get_proc_username(process),
"status": _get_proc_status(process),
"pid": _get_proc_pid(process),
"create_time": _get_proc_create_time(process),
"cpu": {},
"mem": {},
}
try:
for key, value in six.iteritems(process.cpu_times()._asdict()):
info["cpu"][key] = value
for key, value in six.iteritems(process.memory_info()._asdict()):
info["mem"][key] = value
except psutil.NoSuchProcess:
# Process ended since psutil.pids() was run earlier in this
# function. Ignore this process and do not include this process in
# the return data.
continue
result.append(info)
# Stop gathering process info since we've reached the desired number
if len(result) >= num_processes:
break
return result
def is_pid_healthy(pid):
'''
This is a health check that will confirm the PID is running
and executed by salt.
If pusutil is available:
* all architectures are checked
if psutil is not available:
* Linux/Solaris/etc: archs with `/proc/cmdline` available are checked
* AIX/Windows: assume PID is healhty and return True
'''
if HAS_PSUTIL:
try:
proc = psutil.Process(pid)
except psutil.NoSuchProcess:
log.warning("PID %s is no longer running.", pid)
return False
return any(['salt' in cmd for cmd in proc.cmdline()])
if salt.utils.platform.is_aix() or salt.utils.platform.is_windows():
return True
if not salt.utils.process.os_is_running(pid):
log.warning("PID %s is no longer running.", pid)
return False
cmdline_file = os.path.join('proc', str(pid), 'cmdline')
try:
with salt.utils.files.fopen(cmdline_file, 'rb') as fp_:
def shutdown(self, signum=signal.SIGTERM, timeout=10):
'''Shutdown a running daemon'''
children = []
if self.process:
# Last resort, a brute force approach to make sure we leave no child processes running
try:
parent = psutils.Process(self.process.pid)
if hasattr(parent, 'children'):
children = parent.children(recursive=True)
except psutils.NoSuchProcess:
pass
if not self._shutdown and self.process and self.process.returncode == exitcodes.EX_OK:
future = datetime.now() + timedelta(seconds=timeout)
pid = self.wait_for_daemon_pid(timeout)
log.info('Attempting to shutdown "{0}" pid {1} with {2}'.format(
self.name, pid, signum))
while True:
try:
os.kill(pid, signum)
except OSError as err:
if errno.ESRCH == err.errno:
break
raise
if datetime.now() > future:
# One last attempt with a big hammer
try:
pgid = os.getpgid(pid)
os.killpg(pgid, signum)
time.sleep(0.1)
log.warn('Sending SIGKILL to "{0}" pid {1}'.format(
self.name, pid))
os.killpg(pgid, signal.SIGKILL)
time.sleep(0.1)
os.killpg(pgid, signal.SIGKILL)
except OSError as err:
if errno.ESRCH == err.errno:
break
raise TimeoutError(
'Timeout waiting for "{0}" pid {1} to shutdown'.format(
self.name, pid))
time.sleep(0.1)
self.process = None
# Child processes cleanup
if children:
def kill_children():
for child in children[:]:
try:
cmdline = child.cmdline()
log.warning(
'runtests.py left behind the following child process: %s',
cmdline)
child.kill()
children.remove(child)
except psutils.NoSuchProcess:
children.remove(child)
kill_children()
if children:
psutils.wait_procs(children, timeout=5, callback=kill_children)
self._shutdown = True