def test_this_is_really_our_pid():
process = mock.Mock(spec=psutil.Process, username=USER1, pid=PID, create_time=CREATE_TIME)
assert TRH.this_is_really_our_pid(process, process.username, process.create_time)
assert TRH.this_is_really_our_pid(
process, process.username, process.create_time + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) - 1
)
assert not TRH.this_is_really_our_pid(process, "user2", process.create_time)
assert not TRH.this_is_really_our_pid(
process, process.username, process.create_time + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1
)
assert not TRH.this_is_really_our_pid(
process, process.username, process.create_time - (TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1)
)
def write_header(root, sandbox, task_id):
log_dir = os.path.join(sandbox, '.logs')
path = TaskPath(root=root, task_id=task_id, log_dir=log_dir)
header = RunnerHeader(task_id=task_id, sandbox=sandbox, log_dir=log_dir)
ckpt = TaskRunnerHelper.open_checkpoint(path.getpath('runner_checkpoint'))
ckpt.write(RunnerCkpt(runner_header=header))
ckpt.close()
def write_header(root, sandbox, task_id):
log_dir = os.path.join(sandbox, '.logs')
path = TaskPath(root=root, task_id=task_id, log_dir=log_dir)
header = RunnerHeader(task_id=task_id, sandbox=sandbox, log_dir=log_dir)
ckpt = TaskRunnerHelper.open_checkpoint(path.getpath('runner_checkpoint'))
ckpt.write(RunnerCkpt(runner_header=header))
ckpt.close()
def test_scan_process():
# TODO(jon): add more tests for successful cases; this really just looks for errors.
assert TRH.scan_process(
make_runner_state(cpid=None, pid=None), PROCESS_NAME) == (None, None, set())
with mock.patch(PSUTIL_PATH) as p_mock:
class WrappedNoSuchProcess(psutil.NoSuchProcess):
# psutil.NoSuchProcess exception requires an argument, but mock doesn't support that.
def __init__(self): pass
p_mock.side_effect = WrappedNoSuchProcess
assert TRH.scan_process(
make_runner_state(cpid=None), PROCESS_NAME) == (None, None, set())
assert TRH.scan_process(
make_runner_state(pid=None), PROCESS_NAME) == (None, None, set())
with mock.patch(PSUTIL_PATH) as p_mock:
p_mock.side_effect = psutil.Error
assert TRH.scan_process(
make_runner_state(cpid=None), PROCESS_NAME) == (None, None, set())
assert TRH.scan_process(
make_runner_state(pid=None), PROCESS_NAME) == (None, None, set())
with mock.patch(TRH_PATH) as trh_mock:
trh_mock.this_is_really_our_pid.returns = False
assert TRH.scan_process(
make_runner_state(), PROCESS_NAME) == (None, None, set())
with mock.patch(TRH_PATH) as trh_mock:
trh_mock.this_is_really_our_pid.sideffect = psutil.Error
assert TRH.scan_process(
make_runner_state(), PROCESS_NAME) == (None, None, set())
def test_pg_is_killed(self):
runner = self.start_runner()
tm = TaskMonitor(runner.tempdir, runner.task_id)
self.wait_until_running(tm)
process_state, run_number = tm.get_active_processes()[0]
assert process_state.process == 'process'
assert run_number == 0
child_pidfile = os.path.join(runner.sandbox, runner.task_id,
'child.txt')
while not os.path.exists(child_pidfile):
time.sleep(0.1)
parent_pidfile = os.path.join(runner.sandbox, runner.task_id,
'parent.txt')
while not os.path.exists(parent_pidfile):
time.sleep(0.1)
with open(child_pidfile) as fp:
child_pid = int(fp.read().rstrip())
with open(parent_pidfile) as fp:
parent_pid = int(fp.read().rstrip())
ps = ProcessProviderFactory.get()
ps.collect_all()
assert parent_pid in ps.pids()
assert child_pid in ps.pids()
assert child_pid in ps.children_of(parent_pid)
with open(os.path.join(runner.sandbox, runner.task_id, 'exit.txt'),
'w') as fp:
fp.write('go away!')
while tm.task_state() is not TaskState.SUCCESS:
time.sleep(0.1)
state = tm.get_state()
assert state.processes['process'][0].state == ProcessState.SUCCESS
# Another test case may have called setup_child_subreaping(). We therefore have to reap any
# terminated re-parented child processes to ensure that we don't list already terminated
# processes (i.e. zombies) in ps.pids() below.
TaskRunnerHelper.reap_children()
ps.collect_all()
assert parent_pid not in ps.pids()
assert child_pid not in ps.pids()
def test_this_is_really_our_pid():
process = mock.Mock(spec=psutil.Process, pid=PID)
process.username.return_value = USER1
process.create_time.return_value = CREATE_TIME
assert TRH.this_is_really_our_pid(process, process.username(),
process.create_time())
assert TRH.this_is_really_our_pid(
process, process.username(),
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) - 1)
assert not TRH.this_is_really_our_pid(process, 'user2',
process.create_time())
assert not TRH.this_is_really_our_pid(
process, process.username(),
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1)
assert not TRH.this_is_really_our_pid(
process, process.username(),
process.create_time() -
(TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1))
def kill(args, options):
"""Kill task(s)
Usage: thermos kill task_id1 [task_id2 ...]
Regular expressions may be used to match multiple tasks.
"""
if not args:
print('Must specify tasks!', file=sys.stderr)
return
matched_tasks = tasks_from_re(args, options.root, state='active')
if not matched_tasks:
print('No active tasks matched.')
return
for task_id in matched_tasks:
print('Killing %s...' % task_id, end='')
TaskRunnerHelper.kill(task_id, options.root, force=True)
print('done.')
def test_garbage_collector(safe_rmtree, safe_delete):
with temporary_dir() as sandbox, temporary_dir(
) as checkpoint_root, temporary_dir() as log_dir:
path = TaskPath(root=checkpoint_root, task_id='test', log_dir=log_dir)
touch(os.path.join(sandbox, 'test_file1'))
touch(os.path.join(sandbox, 'test_file2'))
safe_mkdir(
os.path.dirname(path.given(state='finished').getpath('task_path')))
safe_mkdir(os.path.dirname(path.getpath('runner_checkpoint')))
touch(path.given(state='finished').getpath('task_path'))
header = RunnerHeader(task_id='test', sandbox=sandbox, log_dir=log_dir)
ckpt = TaskRunnerHelper.open_checkpoint(
path.getpath('runner_checkpoint'))
ckpt.write(RunnerCkpt(runner_header=header))
ckpt.close()
gc = TaskGarbageCollector(checkpoint_root, task_id='test')
assert gc._state.header.log_dir == log_dir
assert gc._state.header.sandbox == sandbox
# erase metadata
gc.erase_metadata()
safe_delete.assert_has_calls([
call(path.given(state='finished').getpath('task_path')),
call(path.getpath('runner_checkpoint'))
],
any_order=True)
safe_rmtree.assert_has_calls([call(path.getpath('checkpoint_path'))])
safe_delete.reset_mock()
safe_rmtree.reset_mock()
# erase logs
gc.erase_logs()
safe_rmtree.assert_has_calls([call(log_dir)])
safe_delete.reset_mock()
safe_rmtree.reset_mock()
# erase sandbox
gc.erase_data()
safe_delete.assert_has_calls([
call(os.path.join(sandbox, 'test_file1')),
call(os.path.join(sandbox, 'test_file2'))
],
any_order=True)
safe_rmtree.assert_has_calls([call(sandbox)])
def test_this_is_really_our_pid():
process = mock.Mock(spec=psutil.Process, pid=PID)
process.username.return_value = USER1
process.create_time.return_value = CREATE_TIME
assert TRH.this_is_really_our_pid(
process,
process.username(),
process.create_time())
assert TRH.this_is_really_our_pid(
process,
process.username(),
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) - 1)
assert not TRH.this_is_really_our_pid(
process,
'user2',
process.create_time())
assert not TRH.this_is_really_our_pid(
process,
process.username(),
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1)
assert not TRH.this_is_really_our_pid(
process,
process.username(),
process.create_time() - (TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1))
def test_garbage_collector(safe_rmtree, safe_delete):
with temporary_dir() as sandbox, temporary_dir() as checkpoint_root, temporary_dir() as log_dir:
path = TaskPath(root=checkpoint_root, task_id='test', log_dir=log_dir)
touch(os.path.join(sandbox, 'test_file1'))
touch(os.path.join(sandbox, 'test_file2'))
safe_mkdir(os.path.dirname(path.given(state='finished').getpath('task_path')))
safe_mkdir(os.path.dirname(path.getpath('runner_checkpoint')))
touch(path.given(state='finished').getpath('task_path'))
header = RunnerHeader(task_id='test', sandbox=sandbox, log_dir=log_dir)
ckpt = TaskRunnerHelper.open_checkpoint(path.getpath('runner_checkpoint'))
ckpt.write(RunnerCkpt(runner_header=header))
ckpt.close()
gc = TaskGarbageCollector(checkpoint_root, task_id='test')
assert gc._state.header.log_dir == log_dir
assert gc._state.header.sandbox == sandbox
# erase metadata
gc.erase_metadata()
safe_delete.assert_has_calls([
call(path.given(state='finished').getpath('task_path')),
call(path.getpath('runner_checkpoint'))], any_order=True)
safe_rmtree.assert_has_calls([call(path.getpath('checkpoint_path'))])
safe_delete.reset_mock()
safe_rmtree.reset_mock()
# erase logs
gc.erase_logs()
safe_rmtree.assert_has_calls([call(log_dir)])
safe_delete.reset_mock()
safe_rmtree.reset_mock()
# erase sandbox
gc.erase_data()
safe_delete.assert_has_calls([
call(os.path.join(sandbox, 'test_file1')),
call(os.path.join(sandbox, 'test_file2'))], any_order=True)
safe_rmtree.assert_has_calls([call(sandbox)])
def test_scan_process():
# TODO(jon): add more tests for successful cases; this really just looks for errors.
assert TRH.scan_process(make_runner_state(cpid=None, pid=None),
PROCESS_NAME) == (None, None, set())
with mock.patch(PSUTIL_PATH) as p_mock:
class WrappedNoSuchProcess(psutil.NoSuchProcess):
# psutil.NoSuchProcess exception requires an argument, but mock doesn't support that.
def __init__(self):
pass
p_mock.side_effect = WrappedNoSuchProcess
assert TRH.scan_process(make_runner_state(cpid=None),
PROCESS_NAME) == (None, None, set())
assert TRH.scan_process(make_runner_state(pid=None),
PROCESS_NAME) == (None, None, set())
with mock.patch(PSUTIL_PATH) as p_mock:
p_mock.side_effect = psutil.Error
assert TRH.scan_process(make_runner_state(cpid=None),
PROCESS_NAME) == (None, None, set())
assert TRH.scan_process(make_runner_state(pid=None),
PROCESS_NAME) == (None, None, set())
with mock.patch(TRH_PATH) as trh_mock:
trh_mock.this_is_really_our_pid.returns = False
assert TRH.scan_process(make_runner_state(),
PROCESS_NAME) == (None, None, set())
with mock.patch(TRH_PATH) as trh_mock:
trh_mock.this_is_really_our_pid.sideffect = psutil.Error
assert TRH.scan_process(make_runner_state(),
PROCESS_NAME) == (None, None, set())
def test_this_is_really_our_pid():
process = mock_process(PID, USER1, uid=UID)
assert TRH.this_is_really_our_pid(process, UID, USER1,
process.create_time())
assert TRH.this_is_really_our_pid(
process, UID, USER1,
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) - 1)
assert TRH.this_is_really_our_pid(
process, UID, 'user2', process.create_time()), (
'An equivalent UID is considered the same user.')
assert not TRH.this_is_really_our_pid(
process, UID + 1, USER1,
process.create_time()), ('UIDs should not match.')
assert not TRH.this_is_really_our_pid(
process, UID, USER1,
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) +
1), ('Process is outside of start time drift.')
assert not TRH.this_is_really_our_pid(
process, UID, USER1,
process.create_time() -
(TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1)), (
'Process is outside of start time drift.')
assert not TRH.this_is_really_our_pid(
process, None, 'user2', process.create_time()
), ("If no uid is checkpointed but the username is different, we can't know it's ours."
)
process = mock_process(PID, USER1)
assert not TRH.this_is_really_our_pid(
process, UID, USER1, process.create_time()
), ('We cannot validate whether this is our process without a process UID.'
)
assert TRH.this_is_really_our_pid(
process, None, USER1,
process.create_time()), ('Previous behavior is preserved.')
process = mock_process(PID, username=KeyError('Unknown user'), uid=UID)
assert TRH.this_is_really_our_pid(process, UID, USER1,
process.create_time())
assert not TRH.this_is_really_our_pid(process, None, USER1,
process.create_time())
assert TRH.this_is_really_our_pid(
process, UID, 'user2', process.create_time()
), ('If the user has been renamed but the UID is the same, this is still our process.'
)
def test_this_is_really_our_pid_root():
# Test the case where a process has a non root uid but the uid in the header
# was root (meaning that it `setuid` itself)
process = mock_process(PID, USER1, uid=UID)
assert TRH.this_is_really_our_pid(process, ROOT_UID, USER1,
process.create_time())
def test_this_is_really_our_pid():
process = mock_process(PID, USER1, uid=UID)
assert TRH.this_is_really_our_pid(process, UID, USER1, process.create_time())
assert TRH.this_is_really_our_pid(process, UID, USER1,
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) - 1)
assert TRH.this_is_really_our_pid(process, UID, 'user2', process.create_time()), (
'An equivalent UID is considered the same user.')
assert not TRH.this_is_really_our_pid(process, UID + 1, USER1, process.create_time()), (
'UIDs should not match.')
assert not TRH.this_is_really_our_pid(process, UID, USER1,
process.create_time() + TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1), (
'Process is outside of start time drift.')
assert not TRH.this_is_really_our_pid(process, UID, USER1,
process.create_time() - (TRH.MAX_START_TIME_DRIFT.as_(Time.SECONDS) + 1)), (
'Process is outside of start time drift.')
assert not TRH.this_is_really_our_pid(process, None, 'user2', process.create_time()), (
"If no uid is checkpointed but the username is different, we can't know it's ours.")
process = mock_process(PID, USER1)
assert not TRH.this_is_really_our_pid(process, UID, USER1, process.create_time()), (
'We cannot validate whether this is our process without a process UID.')
assert TRH.this_is_really_our_pid(process, None, USER1, process.create_time()), (
'Previous behavior is preserved.')
process = mock_process(PID, username=KeyError('Unknown user'), uid=UID)
assert TRH.this_is_really_our_pid(process, UID, USER1, process.create_time())
assert not TRH.this_is_really_our_pid(process, None, USER1, process.create_time())
assert TRH.this_is_really_our_pid(process, UID, 'user2', process.create_time()), (
'If the user has been renamed but the UID is the same, this is still our process.')
def test_this_is_really_our_pid_root(): # Test the case where a process has a non root uid but the uid in the header # was root (meaning that it `setuid` itself) process = mock_process(PID, USER1, uid=UID) assert TRH.this_is_really_our_pid(process, ROOT_UID, USER1, process.create_time())