def _kill(process: psutil.Process, timeout=5) -> bool:
"""Attempts to kill the process in a fail-safe manner and returns status
Tries to terminate the process first and waits for it for the given timeout.
If termination fails - attempts to kill the process instead and wait for it to die.
Args:
process(Process): the process that is to be killed
timeout(int): the timeout to wait after each terminate/kill attempt
Returns:
obj(bool): True if the process has been killed/terminated,
False if the process is still running after this method returns
"""
if not process.is_running():
return True
try:
process.terminate()
try:
process.wait(timeout)
return True
except:
process.kill()
try:
process.wait(timeout)
return True
except:
pass
except:
pass
return not process.is_running()
class Proc(object):
def __init__(self, pid):
self.pid = pid
self.proc = Process(pid)
with self.proc.oneshot():
self.name = self.proc.name()
self.owner = self.proc.username()
self.created = self.proc.create_time()
def poll(self, delay=1):
if not self.proc.is_running():
return ( self.pid, '**DEAD**' + self.name, self.owner, 0, 0 )
with self.proc.oneshot():
self.poll1 = self.proc.cpu_times()
self.virtual1 = self.proc.memory_info().vms
self.system1 = cpu_times()
time.sleep(delay)
with self.proc.oneshot():
self.poll2 = self.proc.cpu_times()
self.virtual2 = self.proc.memory_info().vms
self.system2 = cpu_times()
self.proc_time = sum(self.poll2) - sum(self.poll1)
self.cpu_time = sum(self.system2) - sum(self.system1)
self.virtual = MB(( self.virtual1 + self.virtual2 ) / 2)
self.cpu_percent = 100 * ( self.proc_time / self.cpu_time )
return ( self.pid, self.name, self.owner, self.cpu_percent, self.virtual )
def is_running(self):
return self.proc.is_running()
def __repr__(self):
return "**process** %s (%d)" % ( self.proc.name(), self.pid )
def __str__(self):
return template.format(self.pid, self.name, self.owner, self.cpu_percent, self.virtual)
def test__sample_usage__works():
import time
from psutil import Process
from processwrapper import run_process
with run_process(
'python bin/sample_background_process.py parent') as process:
sample_process = Process(process.pid)
assert sample_process.is_running()
time.sleep(2)
assert not sample_process.is_running()
def _monitor_pid(pid: int, usage: Usage):
proc = Process(pid)
while proc.is_running() and not Executor.is_shutting_down():
usage.cpu_time = sum(proc.cpu_times())
usage.mem_peak = max(usage.mem_peak, proc.memory_info().vms)
time.sleep(0.5)
def is_process_alive(
self,
process: psutil.Process,
) -> bool:
if not process.is_running():
self.logger.info(msg='process status: not running', )
return False
try:
process_status = process.status()
except psutil.NoSuchProcess:
self.logger.info(msg='pid does not exist anymore', )
return False
if process_status in [
psutil.STATUS_DEAD,
psutil.STATUS_ZOMBIE,
]:
self.logger.error(
msg=f'process became a zombie/dead process: {process_status}',
)
return False
return True
def test_cleanup_children_on_terminate(self):
"""
Subprocesses spawned by tasks should be terminated on terminate
"""
class HangingSubprocessTask(luigi.Task):
def run(self):
python = sys.executable
check_call([python, '-c', 'while True: pass'])
task = HangingSubprocessTask()
queue = mock.Mock()
worker_id = 1
task_process = TaskProcess(task, worker_id, queue, lambda: None,
lambda: None)
task_process.start()
parent = Process(task_process.pid)
while not parent.children():
# wait for child process to startup
sleep(0.01)
[child] = parent.children()
task_process.terminate()
child.wait(timeout=1.0) # wait for terminate to complete
self.assertFalse(parent.is_running())
self.assertFalse(child.is_running())
def test_cleanup_children_on_terminate(self):
"""
Subprocesses spawned by tasks should be terminated on terminate
"""
class HangingSubprocessTask(luigi.Task):
def run(self):
python = sys.executable
check_call([python, '-c', 'while True: pass'])
task = HangingSubprocessTask()
queue = mock.Mock()
worker_id = 1
task_process = TaskProcess(task, worker_id, queue, lambda: None, lambda: None)
task_process.start()
parent = Process(task_process.pid)
while not parent.children():
# wait for child process to startup
sleep(0.01)
[child] = parent.children()
task_process.terminate()
child.wait(timeout=1.0) # wait for terminate to complete
self.assertFalse(parent.is_running())
self.assertFalse(child.is_running())
def __watch_process_worker(self, proc: Process):
if proc.is_running():
proc.wait()
log(f"Process: {proc.pid} exited")
else:
log(f"Process: {proc.pid} does not exist (already exited?)")
if proc in self.registred_games:
self._unregister_game(self.process, proc)
def stop_diamond(conf_path):
config_file = os.path.join(conf_path, CONFIG_NAME)
pid = get_pid(config_file)
if pid:
diamond_process = Process(pid)
diamond_process.terminate()
diamond_process.wait(timeout=DEFAULT_TIMEOUT)
if diamond_process.is_running():
raise exceptions.NonRecoverableError("Diamond couldn't be killed")
else:
raise exceptions.NonRecoverableError('Failed reading diamond pid file')
def run_check(self):
"""Runs a check if the pid exists and has not finished yet.
:return: True if the component is running, otherwise returns a generated ``LocalCrashEvent``
:rtype bool or LocalCrashEvent
"""
try:
proc = Process(self.pid)
if proc.is_running():
return True
except NoSuchProcess:
pass
return events.CrashEvent(self.comp_id)
def run_check(self):
try:
proc = Process(self.pid)
if proc.is_running():
return True
except NoSuchProcess:
pass
self.host_lock.acquire()
self.host_status[self.hostname] = None
self.host_lock.release()
return events.DisconnectEvent(self.hostname)
def _rkill(process: psutil.Process, *, dry=True) -> List[KillVictim]:
victims = []
if process.is_running():
with process.oneshot():
ppid = process.ppid()
pid = process.pid
name = process.name()
for child_process in process.children(recursive=True):
victims.extend(_rkill(child_process, dry=dry))
victims.append(KillVictim(ppid, pid, name, ("SKIPPED" if dry else _kill(process))))
return victims
def __init__(self, proc: psutil.Process, proctable):
"""
Class constructor
"""
_dead = False
self._children = list()
self._parent = 0
parent = None
self._proc, self._pt = proc, proctable
try:
self._pgid = os.getpgid(proc.pid)
except:
self._pgid = 0
_dead = True
if not _dead:
parent = proc.parent()
if parent:
self._parent = parent.pid
if not _dead:
self._children = [ p.pid for p in proc.children() ]
with proc.oneshot():
if proc.is_running():
self.rss = proc.memory_info().rss
self.vms = proc.memory_info().vms
self.ctx_vol = proc.num_ctx_switches().voluntary
self.ctx_invol = proc.num_ctx_switches().involuntary
self._cmdline = proc.cmdline()
self.pcpu = None
#self.pcpu += proc.cpu_percent(interval=DEFAULT_INTERVAL)
else:
self.rss = 0
self.vms = 0
self.ctx_vol = 0
self.ctx_invol = 0
self.cmdline = [ ]
self.pcpu = 0.0
def cancel_download(path):
if os.path.exists(DOWNLOAD_HISTORY):
with open(DOWNLOAD_HISTORY, 'rb') as fp:
download_history = pickle.load(fp)
else:
print('Problem managing processes...')
if path in download_history.keys():
pid = download_history[path].pid
try:
p = ProcessManager(pid)
if p.is_running():
p.kill()
except NoSuchProcess as e:
print(e)
if os.path.exists(
os.path.join(DOWNLOAD_LOCATION, '{}incomplete'.format(path))):
os.remove(os.path.join(DOWNLOAD_LOCATION, '{}incomplete'.format(path)))
if os.path.exists(os.path.join(DOWNLOAD_LOCATION, path)):
os.remove(os.path.join(DOWNLOAD_LOCATION, path))
return jsonify({'status': 'success'})
def test_cleanup_children_on_terminate(self):
"""
Subprocesses spawned by tasks should be terminated on terminate
"""
task = HangingSubprocessTask()
queue = mock.Mock()
worker_id = 1
task_process = TaskProcess(task, worker_id, queue)
task_process.start()
parent = Process(task_process.pid)
while not parent.children():
# wait for child process to startup
sleep(0.01)
[child] = parent.children()
task_process.terminate()
child.wait(timeout=1.0) # wait for terminate to complete
self.assertFalse(parent.is_running())
self.assertFalse(child.is_running())
def stop_diamond(conf_path):
config_file = os.path.join(conf_path, CONFIG_NAME)
pid = get_pid(config_file)
if pid:
need_kill = True
try:
diamond_process = Process(pid)
diamond_process.terminate()
diamond_process.wait(timeout=DEFAULT_TIMEOUT)
need_kill = diamond_process.is_running()
except Error:
pass
if need_kill:
call(["sudo", "kill", str(pid)])
# diamond deletes the pid file, even if killed
for _ in range(DEFAULT_TIMEOUT):
pid = get_pid(config_file)
if not pid:
return
sleep(1)
else:
raise exceptions.NonRecoverableError('Failed reading diamond pid file')
def test_cleanup_children_on_terminate(self):
"""
Subprocesses spawned by tasks should be terminated on terminate
"""
task = HangingSubprocessTask()
queue = mock.Mock()
worker_id = 1
task_process = TaskProcess(task, worker_id, queue)
task_process.start()
parent = Process(task_process.pid)
while not parent.children():
# wait for child process to startup
sleep(0.01)
[child] = parent.children()
task_process.terminate()
child.wait(timeout=1.0) # wait for terminate to complete
self.assertFalse(parent.is_running())
self.assertFalse(child.is_running())
def stop_diamond(conf_path):
config_file = os.path.join(conf_path, CONFIG_NAME)
pid = get_pid(config_file)
if pid:
need_kill = True
try:
diamond_process = Process(pid)
diamond_process.terminate()
diamond_process.wait(timeout=DEFAULT_TIMEOUT)
need_kill = diamond_process.is_running()
except Error:
pass
if need_kill:
call(["sudo", "kill", str(pid)])
# diamond deletes the pid file, even if killed
for _ in range(DEFAULT_TIMEOUT):
pid = get_pid(config_file)
if not pid:
return
sleep(1)
else:
raise exceptions.NonRecoverableError('Failed reading diamond pid file')
def stop_capture():
process = Process(pupil_capture.pid)
if process.is_running():
for proc in process.children(recursive=True):
proc.kill()
process.kill()
def _profile_process_with_dir(
self, process: Process, storage_dir_host: str,
process_root: str) -> Optional[Mapping[str, int]]:
output_path_host = os.path.join(
storage_dir_host, f"async-profiler-{process.pid}.output")
touch_path(
output_path_host,
0o666) # make it writable for all, so target process can write
output_path_process = remove_prefix(output_path_host, process_root)
libasyncprofiler_path_host = os.path.join(storage_dir_host,
"libasyncProfiler.so")
libasyncprofiler_path_process = remove_prefix(
libasyncprofiler_path_host, process_root)
if not os.path.exists(libasyncprofiler_path_host):
shutil.copy(resource_path("java/libasyncProfiler.so"),
libasyncprofiler_path_host)
# explicitly chmod to allow access for non-root users
os.chmod(libasyncprofiler_path_host, 0o755)
log_path_host = os.path.join(storage_dir_host,
f"async-profiler-{process.pid}.log")
touch_path(
log_path_host,
0o666) # make it writable for all, so target process can write
log_path_process = remove_prefix(log_path_host, process_root)
free_disk = psutil.disk_usage(output_path_host).free
if free_disk < 250 * 1024:
raise Exception(f"Not enough free disk space: {free_disk}kb")
profiler_event = "itimer" if self._use_itimer else "cpu"
try:
self.run_async_profiler(
self.get_async_profiler_start_cmd(
process.pid,
profiler_event,
self._interval,
output_path_process,
resource_path("java/jattach"),
libasyncprofiler_path_process,
log_path_process,
),
log_path_host,
)
except CalledProcessError:
is_loaded = f" {libasyncprofiler_path_process}" in Path(
f"/proc/{process.pid}/maps").read_text()
logger.warning(
f"async-profiler DSO was{'' if is_loaded else ' not'} loaded into {process.pid}"
)
raise
self._stop_event.wait(self._duration)
if process.is_running():
self.run_async_profiler(
self.get_async_profiler_stop_cmd(
process.pid,
output_path_process,
resource_path("java/jattach"),
libasyncprofiler_path_process,
log_path_process,
),
log_path_host,
)
if self._stop_event.is_set():
raise StopEventSetException()
logger.info(f"Finished profiling process {process.pid}")
return parse_collapsed(Path(output_path_host).read_text())
