def test_neg_self_dependency(self):
"""Tasks cannot depend on themselves. Adding edges where from==to
does nothing. """
wf = jetstream.Workflow()
t = wf.new_task(name='task', after='task')
deps = set(wf.graph.predecessors(t))
self.assertEqual(deps, set())
def test_add_task_w_cmd(self):
wf = jetstream.Workflow()
cmd = 'echo hello world'
t1 = wf.new_task(name='task', cmd=cmd)
t2 = wf['task']
self.assertEqual(t1, t2)
self.assertIs(t1, t2)
def test_predecessors_1(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1', output='log1.txt')
t2 = wf.new_task(name='task2', output='log2.txt')
t3 = wf.new_task(name='task3', input=['log1.txt', 'log2.txt'])
deps = set(wf.graph.predecessors(t3))
self.assertEqual(deps, {t1, t2})
def test_add_task_w_stdout_stderr(self):
wf = jetstream.Workflow()
stdout = 'out.txt'
stderr = 'err.txt'
task = wf.new_task(name='task', stdout=stdout, stderr=stderr)
self.assertEqual(task.directives.get('stdout'), stdout)
self.assertEqual(task.directives.get('stderr'), stderr)
def test_stress_add_tasks_transaction(self):
wf = jetstream.Workflow()
for i in range(500):
wf.new_task(name=f'task{i}')
self.assertEqual(len(wf), 500)
def test_runner(self):
runner = jetstream.Runner()
wf = jetstream.Workflow()
t = wf.new_task(cmd='hostname', stdout='/dev/null')
runner.start(workflow=wf)
self.assertTrue(t.is_complete())
def test_pop_task(self):
t = jetstream.Task()
wf = jetstream.Workflow(tasks=[
t,
])
t2 = wf.pop(t.name)
self.assertIs(t, t2)
self.assertEqual(len(wf), 0)
def test_add_task_w_before(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1')
t2 = wf.new_task(name='task2', before='task1')
deps = set(wf.graph.predecessors(t1))
self.assertEqual(deps, {
t2,
})
def test_add_task_w_after(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1', )
t2 = wf.new_task(name='task2', after='task1')
deps = set(wf.graph.predecessors(t2))
self.assertEqual(deps, {
t1,
})
def test_successors_1(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1', output='banana.txt')
t2 = wf.new_task(name='task2', input='banana.txt')
deps = set(wf.graph.successors(t1))
self.assertEqual(deps, {
t2,
})
def test_dependent_failure(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='hello')
t2 = wf.new_task(name='goodbye', after='hello')
wf.graph.skip_descendants(t1)
self.assertTrue(t2.is_done())
self.assertTrue(t2.is_failed())
self.assertTrue(t2.is_skipped())
self.assertFalse(t2.is_complete())
def test_successors_2(self):
"""Test adding input/output directives as lists"""
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1', output=['banana.txt', 'banana2.txt'])
t2 = wf.new_task(name='task2', input=['banana.txt', 'banana2.txt'])
deps = set(wf.graph.successors(t1))
self.assertEqual(deps, {
t2,
})
def test_reset_parents(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='taskA')
t2 = wf.new_task(name='taskB', after='taskA', reset='parents')
for t in wf:
t.complete()
self.assertEqual(t1.status, 'complete')
wf.reset_task(t2)
self.assertEqual(t1.status, 'new')
def test_reset_task_name_neg(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='taskA')
t2 = wf.new_task(name='taskB', after='taskA')
t3 = wf.new_task(name='taskC', after='taskB', reset='taskZ')
for t in wf:
t.complete()
self.assertEqual(t1.status, 'complete')
with self.assertRaises(KeyError) as ctx:
wf.reset_task(t3)
def load_workflow(render):
"""Given a rendered template string, loads the tasks and returns a workflow"""
log.debug(f'Parsing tasks from render:\n{render}')
tasks = jetstream.utils.parse_yaml(render)
if not tasks:
raise ValueError('No tasks found!')
log.info('Loading tasks...')
if isinstance(tasks, Mapping):
# If template is a mapping, it is expected to have a tasks section
props = {k: v for k, v in tasks.items() if k != 'tasks'}
tasks = [jetstream.Task(**t) for t in tasks['tasks']]
wf = jetstream.Workflow(tasks=tasks, props=props)
else:
tasks = [jetstream.Task(**t) for t in tasks]
wf = jetstream.Workflow(tasks=tasks)
wf.reload_graph()
return wf
def test_graph_iter(self):
"""While tasks are pending but workflow is not complete, the
WorkflowGraphIterator will return None. When all tasks are complete
it should raise StopIteration"""
wf = jetstream.Workflow()
t = wf.new_task(name='task')
i = iter(wf.graph)
self.assertIs(next(i), t)
self.assertIs(next(i), None)
self.assertIs(next(i), None)
t.complete()
self.assertRaises(StopIteration, next, i)
def test_load(self):
wf = jetstream.Workflow()
# Add some tasks
for i in range(100):
wf.new_task(name=f'task{i}', cmd='echo task {}'.format(i))
# Modify some state
wf['task50'].complete()
wf['task42'].skip(foo='bar')
jetstream.save_workflow(wf, 'wf.pickle')
wf2 = jetstream.load_workflow('wf.pickle')
self.assertEqual(len(wf), 100)
self.assertIn('task1', wf2)
self.assertTrue(wf2['task50'].is_complete())
self.assertEqual(wf2['task42'].state.get('foo'), 'bar')
def init(path=None, config=None, id=None):
"""Sets up a new project dir. This will overwrite jetstream/project.yaml if
the project already exists. """
if path is None:
path = os.getcwd()
paths = ProjectPaths(path)
os.makedirs(paths.index_dir, exist_ok=True)
os.makedirs(paths.logs_dir, exist_ok=True)
os.makedirs(paths.history_dir, exist_ok=True)
fingerprint = jetstream.utils.Fingerprint(note='init', id=id).to_dict()
config = config or dict()
config.update(__project__=fingerprint)
with open(paths.index_path, 'w') as fp:
jetstream.utils.dump_yaml(config, fp)
wf = jetstream.Workflow()
wf.save(paths.workflow_path)
return Project(path)
def load_workflow(self):
try:
return jetstream.load_workflow(self.paths.workflow_path)
except FileNotFoundError:
return jetstream.Workflow(path=self.paths.workflow_path)
def test_add_task_w_mem(self):
wf = jetstream.Workflow()
mem = 4
task = wf.new_task(name='task', mem=mem)
self.assertEqual(task.directives.get('mem'), mem)
def test_add_task_w_cpus(self):
wf = jetstream.Workflow()
cpus = 4
task = wf.new_task(name='task', cpus=cpus)
self.assertEqual(task.directives.get('cpus'), cpus)
def test_init(self):
jetstream.Workflow()
def test_add_task_w_stdin(self):
wf = jetstream.Workflow()
stdin = '/path/to/stdin.txt'
task = wf.new_task(name='task', stdin=stdin)
self.assertEqual(task.directives.get('stdin'), stdin)
def test_is_ready(self):
wf = jetstream.Workflow()
t1 = wf.new_task(name='task1')
t2 = wf.new_task(name='task2', after='task1')
self.assertEqual(wf.graph.is_ready(t1), True)
self.assertEqual(wf.graph.is_ready(t2), False)
def test_indexing(self):
wf = jetstream.Workflow()
t1 = wf.new_task()
t2 = wf[t1.name]
t3 = wf[t1.name]
self.assertIs(t1, t2, t3)
def test_add_duplicate_task(self):
wf = jetstream.Workflow()
t1 = Task()
t2 = Task()
wf.add(t1)
self.assertRaises(ValueError, wf.add, t2)
def test_new_task(self):
wf = jetstream.Workflow()
wf.new_task(cmd='hello world')
self.assertEqual(len(wf), 1)
def test_save(self):
wf = jetstream.Workflow()
for i in range(100):
wf.new_task(name=f'task{i}', cmd='echo task {}'.format(i))
jetstream.save_workflow(wf, 'wf.pickle')
self.assertTrue(os.path.exists('wf.pickle'))
def test_len(self):
wf = jetstream.Workflow()
self.assertEqual(len(wf), 0)
def test_add_task(self):
t = jetstream.Task()
wf = jetstream.Workflow()
wf.add(t)
