def test_process_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
os.kill(process_state.pid, signal.SIGKILL)
while True:
if not hasattr(runner, 'state'):
time.sleep(0.1)
else:
break
assert runner.state.statuses[-1].state == TaskState.SUCCESS
assert 'process' in runner.state.processes
assert len(runner.state.processes['process']) == 2
assert runner.state.processes['process'][
0].state == ProcessState.KILLED
assert runner.state.processes['process'][
0].return_code == -signal.SIGKILL
assert runner.state.processes['process'][
1].state == ProcessState.SUCCESS
def test_coordinator_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
os.kill(process_state.coordinator_pid, signal.SIGKILL)
while True:
active_procs = tm.get_active_processes()
if active_procs and active_procs[0][1] > 0:
break
time.sleep(0.2)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 1
os.kill(process_state.pid, signal.SIGKILL)
while True:
active_procs = tm.get_active_processes()
if active_procs and active_procs[0][1] > 1:
break
time.sleep(0.2)
self.wait_until_running(tm)
os.kill(runner.po.pid, signal.SIGKILL)
try:
state = tm.get_state()
assert state.processes['ignorant_process'][
0].state == ProcessState.LOST
assert state.processes['ignorant_process'][
1].state == ProcessState.KILLED
assert state.processes['ignorant_process'][
2].state == ProcessState.RUNNING
finally:
os.kill(state.processes['ignorant_process'][2].coordinator_pid,
signal.SIGKILL)
os.kill(state.processes['ignorant_process'][2].pid, signal.SIGKILL)
def test_coordinator_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.pathspec, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
os.kill(process_state.coordinator_pid, signal.SIGKILL)
while True:
active_procs = tm.get_active_processes()
if active_procs and active_procs[0][1] > 0:
break
time.sleep(0.2)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 1
os.kill(process_state.pid, signal.SIGKILL)
while True:
active_procs = tm.get_active_processes()
if active_procs and active_procs[0][1] > 1:
break
time.sleep(0.2)
self.wait_until_running(tm)
os.kill(runner.po.pid, signal.SIGKILL)
try:
state = tm.get_state()
assert state.processes['ignorant_process'][0].state == ProcessState.LOST
assert state.processes['ignorant_process'][1].state == ProcessState.KILLED
assert state.processes['ignorant_process'][2].state == ProcessState.RUNNING
finally:
os.kill(state.processes['ignorant_process'][2].coordinator_pid, signal.SIGKILL)
os.kill(state.processes['ignorant_process'][2].pid, signal.SIGKILL)
def test_pg_is_killed(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
child_pidfile = os.path.join(runner.sandbox, runner.task_id,
'child.txt')
while not os.path.exists(child_pidfile):
time.sleep(0.1)
parent_pidfile = os.path.join(runner.sandbox, runner.task_id,
'parent.txt')
while not os.path.exists(parent_pidfile):
time.sleep(0.1)
with open(child_pidfile) as fp:
child_pid = int(fp.read().rstrip())
with open(parent_pidfile) as fp:
parent_pid = int(fp.read().rstrip())
ps = ProcessProviderFactory.get()
ps.collect_all()
assert parent_pid in ps.pids()
assert child_pid in ps.pids()
assert child_pid in ps.children_of(parent_pid)
with open(os.path.join(runner.sandbox, runner.task_id, 'exit.txt'),
'w') as fp:
fp.write('go away!')
while tm.task_state() is not TaskState.SUCCESS:
time.sleep(0.1)
state = tm.get_state()
assert state.processes['process'][0].state == ProcessState.SUCCESS
# Another test case may have called setup_child_subreaping(). We therefore have to reap any
# terminated re-parented child processes to ensure that we don't list already terminated
# processes (i.e. zombies) in ps.pids() below.
TaskRunnerHelper.reap_children()
ps.collect_all()
assert parent_pid not in ps.pids()
assert child_pid not in ps.pids()
def test_coordinator_dead_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.pathspec, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
os.kill(runner.po.pid, signal.SIGKILL)
os.kill(process_state.coordinator_pid, signal.SIGKILL)
os.kill(process_state.pid, signal.SIGKILL)
killer = TaskRunner.get(runner.task_id, runner.root)
assert killer is not None
killer.kill(force=True)
state = tm.get_state()
assert len(state.processes['ignorant_process']) == 1
assert state.processes['ignorant_process'][0].state == ProcessState.LOST
def test_coordinator_dead_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
os.kill(runner.po.pid, signal.SIGKILL)
os.kill(process_state.coordinator_pid, signal.SIGKILL)
os.kill(process_state.pid, signal.SIGKILL)
killer = TaskRunner.get(runner.task_id, runner.root)
assert killer is not None
killer.kill(force=True)
state = tm.get_state()
assert len(state.processes['ignorant_process']) == 1
assert state.processes['ignorant_process'][
0].state == ProcessState.LOST
def test_preemption_wait(self):
runner = self.start_runner()
tm = TaskMonitor(runner.pathspec, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
preempter = TaskRunner.get(runner.task_id, runner.root)
assert preempter is not None
now = time.time()
preempter.kill(force=True, preemption_wait=Amount(1, Time.SECONDS))
duration = time.time() - now
# This is arbitrary, but make sure we finish within half a second of
# requested preemption wait.
assert abs(duration - 1.0) < 0.5
assert preempter.state.statuses[-1].state == TaskState.KILLED
assert preempter.state.processes['ignorant_process'][-1].state == ProcessState.KILLED
def test_preemption_wait(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'ignorant_process'
assert run_number == 0
preempter = TaskRunner.get(runner.task_id, runner.root)
assert preempter is not None
now = time.time()
preempter.kill(force=True, preemption_wait=Amount(1, Time.SECONDS))
duration = time.time() - now
# This is arbitrary, but make sure we finish within half a second of
# requested preemption wait.
assert abs(duration - 1.0) < 0.5
assert preempter.state.statuses[-1].state == TaskState.KILLED
assert preempter.state.processes['ignorant_process'][
-1].state == ProcessState.KILLED
def test_coordinator_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.pathspec, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
os.kill(process_state.coordinator_pid, signal.SIGKILL)
while True:
if not hasattr(runner, 'state'):
time.sleep(0.1)
else:
break
assert runner.state.statuses[-1].state == TaskState.SUCCESS
assert 'process' in runner.state.processes
assert len(runner.state.processes['process']) == 2
assert runner.state.processes['process'][0].state == ProcessState.LOST
assert runner.state.processes['process'][1].state == ProcessState.SUCCESS
def test_pg_is_killed(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
child_pidfile = os.path.join(runner.sandbox, runner.task_id,
'child.txt')
while not os.path.exists(child_pidfile):
time.sleep(0.1)
parent_pidfile = os.path.join(runner.sandbox, runner.task_id,
'parent.txt')
while not os.path.exists(parent_pidfile):
time.sleep(0.1)
with open(child_pidfile) as fp:
child_pid = int(fp.read().rstrip())
with open(parent_pidfile) as fp:
parent_pid = int(fp.read().rstrip())
ps = ProcessProviderFactory.get()
ps.collect_all()
assert parent_pid in ps.pids()
assert child_pid in ps.pids()
assert child_pid in ps.children_of(parent_pid)
with open(os.path.join(runner.sandbox, runner.task_id, 'exit.txt'),
'w') as fp:
fp.write('go away!')
while tm.task_state() is not TaskState.SUCCESS:
time.sleep(0.1)
state = tm.get_state()
assert state.processes['process'][0].state == ProcessState.SUCCESS
ps.collect_all()
assert parent_pid not in ps.pids()
assert child_pid not in ps.pids()
def test_process_kill(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == "process"
assert run_number == 0
os.kill(process_state.pid, signal.SIGKILL)
while True:
if not hasattr(runner, "state"):
time.sleep(0.1)
else:
break
assert runner.state.statuses[-1].state == TaskState.SUCCESS
assert "process" in runner.state.processes
assert len(runner.state.processes["process"]) == 2
assert runner.state.processes["process"][0].state == ProcessState.KILLED
assert runner.state.processes["process"][0].return_code == -signal.SIGKILL
assert runner.state.processes["process"][1].state == ProcessState.SUCCESS
def test_pg_is_killed(self):
runner = self.start_runner()
tm = TaskMonitor(runner.pathspec, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
child_pidfile = os.path.join(runner.sandbox, runner.task_id, 'child.txt')
while not os.path.exists(child_pidfile):
time.sleep(0.1)
parent_pidfile = os.path.join(runner.sandbox, runner.task_id, 'parent.txt')
while not os.path.exists(parent_pidfile):
time.sleep(0.1)
with open(child_pidfile) as fp:
child_pid = int(fp.read().rstrip())
with open(parent_pidfile) as fp:
parent_pid = int(fp.read().rstrip())
ps = ProcessProviderFactory.get()
ps.collect_all()
assert parent_pid in ps.pids()
assert child_pid in ps.pids()
assert child_pid in ps.children_of(parent_pid)
with open(os.path.join(runner.sandbox, runner.task_id, 'exit.txt'), 'w') as fp:
fp.write('go away!')
while tm.task_state() is not TaskState.SUCCESS:
time.sleep(0.1)
state = tm.get_state()
assert state.processes['process'][0].state == ProcessState.SUCCESS
ps.collect_all()
assert parent_pid not in ps.pids()
assert child_pid not in ps.pids()