def test_neg_self_dependency(self): """Tasks cannot depend on themselves. Adding edges where from==to does nothing. """ wf = jetstream.Workflow() t = wf.new_task(name='task', after='task') deps = set(wf.graph.predecessors(t)) self.assertEqual(deps, set())
def test_add_task_w_cmd(self): wf = jetstream.Workflow() cmd = 'echo hello world' t1 = wf.new_task(name='task', cmd=cmd) t2 = wf['task'] self.assertEqual(t1, t2) self.assertIs(t1, t2)
def test_predecessors_1(self): wf = jetstream.Workflow() t1 = wf.new_task(name='task1', output='log1.txt') t2 = wf.new_task(name='task2', output='log2.txt') t3 = wf.new_task(name='task3', input=['log1.txt', 'log2.txt']) deps = set(wf.graph.predecessors(t3)) self.assertEqual(deps, {t1, t2})
def test_add_task_w_stdout_stderr(self): wf = jetstream.Workflow() stdout = 'out.txt' stderr = 'err.txt' task = wf.new_task(name='task', stdout=stdout, stderr=stderr) self.assertEqual(task.directives.get('stdout'), stdout) self.assertEqual(task.directives.get('stderr'), stderr)
def test_stress_add_tasks_transaction(self): wf = jetstream.Workflow() for i in range(500): wf.new_task(name=f'task{i}') self.assertEqual(len(wf), 500)
def test_runner(self): runner = jetstream.Runner() wf = jetstream.Workflow() t = wf.new_task(cmd='hostname', stdout='/dev/null') runner.start(workflow=wf) self.assertTrue(t.is_complete())
def test_pop_task(self): t = jetstream.Task() wf = jetstream.Workflow(tasks=[ t, ]) t2 = wf.pop(t.name) self.assertIs(t, t2) self.assertEqual(len(wf), 0)
def test_add_task_w_before(self): wf = jetstream.Workflow() t1 = wf.new_task(name='task1') t2 = wf.new_task(name='task2', before='task1') deps = set(wf.graph.predecessors(t1)) self.assertEqual(deps, { t2, })
def test_add_task_w_after(self): wf = jetstream.Workflow() t1 = wf.new_task(name='task1', ) t2 = wf.new_task(name='task2', after='task1') deps = set(wf.graph.predecessors(t2)) self.assertEqual(deps, { t1, })
def test_successors_1(self): wf = jetstream.Workflow() t1 = wf.new_task(name='task1', output='banana.txt') t2 = wf.new_task(name='task2', input='banana.txt') deps = set(wf.graph.successors(t1)) self.assertEqual(deps, { t2, })
def test_dependent_failure(self): wf = jetstream.Workflow() t1 = wf.new_task(name='hello') t2 = wf.new_task(name='goodbye', after='hello') wf.graph.skip_descendants(t1) self.assertTrue(t2.is_done()) self.assertTrue(t2.is_failed()) self.assertTrue(t2.is_skipped()) self.assertFalse(t2.is_complete())
def test_successors_2(self): """Test adding input/output directives as lists""" wf = jetstream.Workflow() t1 = wf.new_task(name='task1', output=['banana.txt', 'banana2.txt']) t2 = wf.new_task(name='task2', input=['banana.txt', 'banana2.txt']) deps = set(wf.graph.successors(t1)) self.assertEqual(deps, { t2, })
def test_reset_parents(self): wf = jetstream.Workflow() t1 = wf.new_task(name='taskA') t2 = wf.new_task(name='taskB', after='taskA', reset='parents') for t in wf: t.complete() self.assertEqual(t1.status, 'complete') wf.reset_task(t2) self.assertEqual(t1.status, 'new')
def test_reset_task_name_neg(self): wf = jetstream.Workflow() t1 = wf.new_task(name='taskA') t2 = wf.new_task(name='taskB', after='taskA') t3 = wf.new_task(name='taskC', after='taskB', reset='taskZ') for t in wf: t.complete() self.assertEqual(t1.status, 'complete') with self.assertRaises(KeyError) as ctx: wf.reset_task(t3)
def load_workflow(render): """Given a rendered template string, loads the tasks and returns a workflow""" log.debug(f'Parsing tasks from render:\n{render}') tasks = jetstream.utils.parse_yaml(render) if not tasks: raise ValueError('No tasks found!') log.info('Loading tasks...') if isinstance(tasks, Mapping): # If template is a mapping, it is expected to have a tasks section props = {k: v for k, v in tasks.items() if k != 'tasks'} tasks = [jetstream.Task(**t) for t in tasks['tasks']] wf = jetstream.Workflow(tasks=tasks, props=props) else: tasks = [jetstream.Task(**t) for t in tasks] wf = jetstream.Workflow(tasks=tasks) wf.reload_graph() return wf
def test_graph_iter(self): """While tasks are pending but workflow is not complete, the WorkflowGraphIterator will return None. When all tasks are complete it should raise StopIteration""" wf = jetstream.Workflow() t = wf.new_task(name='task') i = iter(wf.graph) self.assertIs(next(i), t) self.assertIs(next(i), None) self.assertIs(next(i), None) t.complete() self.assertRaises(StopIteration, next, i)
def test_load(self): wf = jetstream.Workflow() # Add some tasks for i in range(100): wf.new_task(name=f'task{i}', cmd='echo task {}'.format(i)) # Modify some state wf['task50'].complete() wf['task42'].skip(foo='bar') jetstream.save_workflow(wf, 'wf.pickle') wf2 = jetstream.load_workflow('wf.pickle') self.assertEqual(len(wf), 100) self.assertIn('task1', wf2) self.assertTrue(wf2['task50'].is_complete()) self.assertEqual(wf2['task42'].state.get('foo'), 'bar')
def init(path=None, config=None, id=None): """Sets up a new project dir. This will overwrite jetstream/project.yaml if the project already exists. """ if path is None: path = os.getcwd() paths = ProjectPaths(path) os.makedirs(paths.index_dir, exist_ok=True) os.makedirs(paths.logs_dir, exist_ok=True) os.makedirs(paths.history_dir, exist_ok=True) fingerprint = jetstream.utils.Fingerprint(note='init', id=id).to_dict() config = config or dict() config.update(__project__=fingerprint) with open(paths.index_path, 'w') as fp: jetstream.utils.dump_yaml(config, fp) wf = jetstream.Workflow() wf.save(paths.workflow_path) return Project(path)
def load_workflow(self): try: return jetstream.load_workflow(self.paths.workflow_path) except FileNotFoundError: return jetstream.Workflow(path=self.paths.workflow_path)
def test_add_task_w_mem(self): wf = jetstream.Workflow() mem = 4 task = wf.new_task(name='task', mem=mem) self.assertEqual(task.directives.get('mem'), mem)
def test_add_task_w_cpus(self): wf = jetstream.Workflow() cpus = 4 task = wf.new_task(name='task', cpus=cpus) self.assertEqual(task.directives.get('cpus'), cpus)
def test_init(self): jetstream.Workflow()
def test_add_task_w_stdin(self): wf = jetstream.Workflow() stdin = '/path/to/stdin.txt' task = wf.new_task(name='task', stdin=stdin) self.assertEqual(task.directives.get('stdin'), stdin)
def test_is_ready(self): wf = jetstream.Workflow() t1 = wf.new_task(name='task1') t2 = wf.new_task(name='task2', after='task1') self.assertEqual(wf.graph.is_ready(t1), True) self.assertEqual(wf.graph.is_ready(t2), False)
def test_indexing(self): wf = jetstream.Workflow() t1 = wf.new_task() t2 = wf[t1.name] t3 = wf[t1.name] self.assertIs(t1, t2, t3)
def test_add_duplicate_task(self): wf = jetstream.Workflow() t1 = Task() t2 = Task() wf.add(t1) self.assertRaises(ValueError, wf.add, t2)
def test_new_task(self): wf = jetstream.Workflow() wf.new_task(cmd='hello world') self.assertEqual(len(wf), 1)
def test_save(self): wf = jetstream.Workflow() for i in range(100): wf.new_task(name=f'task{i}', cmd='echo task {}'.format(i)) jetstream.save_workflow(wf, 'wf.pickle') self.assertTrue(os.path.exists('wf.pickle'))
def test_len(self): wf = jetstream.Workflow() self.assertEqual(len(wf), 0)
def test_add_task(self): t = jetstream.Task() wf = jetstream.Workflow() wf.add(t)