def test_simple_process(): with temporary_dir() as td: taskpath = TaskPath(root=td, task_id='task', process='process', run=0) sandbox = setup_sandbox(td, taskpath) p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox) p.start() rc = wait_for_rc(taskpath.getpath('process_checkpoint')) assert rc == 0 stdout = taskpath.with_filename('stdout').getpath('process_logdir') assert os.path.exists(stdout) with open(stdout, 'r') as fp: assert fp.read() == 'hello world\n'
def test_log_permissions(): with temporary_dir() as td: taskpath = TaskPath(root=td, task_id='task', process='process', run=0) sandbox = setup_sandbox(td, taskpath) p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox) p.start() rc = wait_for_rc(taskpath.getpath('process_checkpoint')) stdout = taskpath.with_filename('stdout').getpath('process_logdir') stderr = taskpath.with_filename('stderr').getpath('process_logdir') assert os.path.exists(stdout) assert os.path.exists(stderr) assert os.stat(stdout).st_uid == os.getuid() assert os.stat(stderr).st_uid == os.getuid()
def test_log_permissions_other_user(*mocks): with temporary_dir() as td: some_user = get_other_nonroot_user() taskpath = TaskPath(root=td, task_id='task', process='process', run=0) sandbox = setup_sandbox(td, taskpath) p = TestProcess('process', 'echo hello world', 0, taskpath, sandbox, user=some_user.pw_name) p.start() rc = wait_for_rc(taskpath.getpath('process_checkpoint')) # since we're not actually root, the best we can do is check the right things were attempted stdout = taskpath.with_filename('stdout').getpath('process_logdir') stderr = taskpath.with_filename('stderr').getpath('process_logdir') assert os.path.exists(stdout) assert os.path.exists(stderr) assert os.chown.calledwith(stdout, some_user.pw_uid, some_user.pw_gid) assert os.chown.calledwith(stderr, some_user.pw_uid, some_user.pw_gid)
class TaskDetector(object): """ Helper class in front of TaskPath to detect active/finished/running tasks. Performs no introspection on the state of a task; merely detects based on file paths on disk. """ class MatchingError(Exception): pass def __init__(self, root): self._root_dir = root self._pathspec = TaskPath() def get_task_ids(self, state=None): paths = glob.glob(self._pathspec.given(root=self._root_dir, task_id="*", state=state or '*') .getpath('task_path')) path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir), task_id="(\S+)", state='(\S+)') .getpath('task_path')) for path in paths: try: task_state, task_id = path_re.match(path).groups() except: continue if state is None or task_state == state: yield (task_state, task_id) def get_process_runs(self, task_id, log_dir): paths = glob.glob(self._pathspec.given(root=self._root_dir, task_id=task_id, log_dir=log_dir, process='*', run='*') .getpath('process_logdir')) path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir), task_id=re.escape(task_id), log_dir=log_dir, process='(\S+)', run='(\d+)') .getpath('process_logdir')) for path in paths: try: process, run = path_re.match(path).groups() except: continue yield process, int(run) def get_process_logs(self, task_id, log_dir): for process, run in self.get_process_runs(task_id, log_dir): for logtype in ('stdout', 'stderr'): path = (self._pathspec.with_filename(logtype).given(root=self._root_dir, task_id=task_id, log_dir=log_dir, process=process, run=run) .getpath('process_logdir')) if os.path.exists(path): yield path def get_checkpoint(self, task_id): return self._pathspec.given(root=self._root_dir, task_id=task_id).getpath('runner_checkpoint') def get_process_checkpoints(self, task_id): matching_paths = glob.glob(self._pathspec.given(root=self._root_dir, task_id=task_id, process='*') .getpath('process_checkpoint')) path_re = re.compile(self._pathspec.given(root=re.escape(self._root_dir), task_id=re.escape(task_id), process='(\S+)') .getpath('process_checkpoint')) for path in matching_paths: try: process, = path_re.match(path).groups() except: continue yield path
class TaskDetector(object): """ Helper class in front of TaskPath to detect active/finished/running tasks. Performs no introspection on the state of a task; merely detects based on file paths on disk. """ class MatchingError(Exception): pass def __init__(self, root): self._root_dir = root self._pathspec = TaskPath() def get_task_ids(self, state=None): paths = glob.glob( self._pathspec.given(root=self._root_dir, task_id="*", state=state or '*').getpath('task_path')) path_re = re.compile( self._pathspec.given(root=re.escape(self._root_dir), task_id="(\S+)", state='(\S+)').getpath('task_path')) for path in paths: try: task_state, task_id = path_re.match(path).groups() except: continue if state is None or task_state == state: yield (task_state, task_id) def get_process_runs(self, task_id, log_dir): paths = glob.glob( self._pathspec.given(root=self._root_dir, task_id=task_id, log_dir=log_dir, process='*', run='*').getpath('process_logdir')) path_re = re.compile( self._pathspec.given(root=re.escape(self._root_dir), task_id=re.escape(task_id), log_dir=log_dir, process='(\S+)', run='(\d+)').getpath('process_logdir')) for path in paths: try: process, run = path_re.match(path).groups() except: continue yield process, int(run) def get_process_logs(self, task_id, log_dir): for process, run in self.get_process_runs(task_id, log_dir): for logtype in ('stdout', 'stderr'): path = (self._pathspec.with_filename(logtype).given( root=self._root_dir, task_id=task_id, log_dir=log_dir, process=process, run=run).getpath('process_logdir')) if os.path.exists(path): yield path def get_checkpoint(self, task_id): return self._pathspec.given( root=self._root_dir, task_id=task_id).getpath('runner_checkpoint') def get_process_checkpoints(self, task_id): matching_paths = glob.glob( self._pathspec.given(root=self._root_dir, task_id=task_id, process='*').getpath('process_checkpoint')) path_re = re.compile( self._pathspec.given( root=re.escape(self._root_dir), task_id=re.escape(task_id), process='(\S+)').getpath('process_checkpoint')) for path in matching_paths: try: process, = path_re.match(path).groups() except: continue yield path