def test__DAG_init():
DAG()
# init with dependencies
make_tea = Task('make_tea.py', 'test-env')
drink_tea = Task('drink_tea.py', 'test-env')
DAG(dependencies={drink_tea: make_tea})
def test__DAG_add_dependencies():
A, B = get_two_tasks()
C = Task('C.py', env='test-env')
dag = DAG()
dag.add_dependencies({B: A})
assert dag._edges[A] == set([B])
dag = DAG()
dag.add_dependencies({C: {A, B}})
assert dag._edges[A] == set([C])
assert dag._edges[B] == set([C])
def test__run_tasks_fail_hard():
from dequindre import DAG, Dequindre
from dequindre.commons import common_task
with common_task('./tea-tasks/{}', 'python') as TeaTask:
boil_water = TeaTask('boil_water.py')
pour_water = TeaTask('pour_water.py')
prep_infuser = TeaTask('prep_infuser.py')
steep_tea = TeaTask('steep_tea.py')
fake_task = TeaTask('not-a-real-task.py')
make_tea = DAG(
dependencies={
pour_water: fake_task,
boil_water: {pour_water},
steep_tea: {boil_water, prep_infuser}
})
## run tasks
dq = Dequindre(make_tea)
with pytest.raises(AssertionError):
dq.run_tasks(error_handling='bad-arg')
with pytest.raises(EarlyAbortError):
dq.run_tasks(error_handling='hard')
def test__DAG_get_upstream():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, depends_on=A)
assert dag.get_upstream() is not None
assert dag.get_upstream()[B] == {A,}
assert dag.get_upstream() == {B: {A,}}, 'Task A is not upstream'
def test__readme_example():
from dequindre import Task, DAG, Dequindre
## define tasks and environments
pour_water = Task('./tea-tasks/pour_water.py')
boil_water = Task('./tea-tasks/boil_water.py')
prep_infuser = Task('./tea-tasks/prep_infuser.py')
steep_tea = Task('./tea-tasks/steep_tea.py')
## define runtime dependencies
make_tea = DAG(dependencies={
boil_water: {pour_water},
steep_tea: {boil_water, prep_infuser}
})
## run tasks
dq = Dequindre(make_tea)
dq.get_schedules()
# defaultdict(<class 'set'>, {
# 1: {Task(prep_infuser.py), Task(pour_water.py)},
# 2: {Task(boil_water.py)},
# 3: {Task(steep_tea.py)}})
## dq.run_tasks() can run the files if they exist.
dq.run_tasks()
def test__Dequindre_get_schedules():
A = Task('A.py', 'test-env')
B = Task('B.py', 'test-env')
C = Task('C.py', 'test-env')
Z = Task('Z.py', 'test-env')
dag = DAG()
dag.add_tasks({A, B, C, Z})
dag.add_dependencies({B: A, C: B})
dq = Dequindre(dag)
priorities = dq.get_schedules()
testable = {}
# build a testable result dict
for k, v in priorities.items():
new_set = set()
if isinstance(v, Task):
testable[k] = set(v)
continue
for vi in v:
new_set.add(hash(vi))
testable[k] = new_set
assert testable == {
1: {hash(A), hash(Z)},
2: {hash(B)},
3: {hash(C)},
}
def test__DAG_get_downstream():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, depends_on=A)
assert dag.get_downstream() is not None
assert dag.get_downstream()[A] == {B,}
assert dag.get_downstream() == {A: {B,}}, 'Task B is not downstream'
def test__DAG_add_task():
A, B = get_two_tasks()
dag = DAG()
dag.add_task(A)
assert dag.tasks == {A,}, 'Test Task was not added to the DAG'
def test__DAG_remove_task():
A, B = get_two_tasks()
dag = DAG()
dag.add_tasks({A, B})
dag.remove_task(A)
assert dag.tasks == {B}
def test__DAG_is_cyclic():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, depends_on=A)
assert not dag.is_cyclic(), 'acyclic graph idenfied as cyclic'
with pytest.raises(CyclicGraphError):
dag.add_dependency(A, depends_on=B)
def test__DAG_remove_tasks():
A, B = get_two_tasks()
C = Task('C.py')
dag = DAG()
dag.add_tasks({A, B, C})
dag.remove_tasks({A, B})
assert dag.tasks == {C}
dag.remove_tasks(C)
assert dag.tasks == set()
def test__DAG_add_tasks():
A, B = get_two_tasks()
C = Task('C.py')
dag = DAG()
dag.add_tasks({A, B})
assert dag.tasks == {A,B}, 'Test Tasks were not added to the DAG'
dag.add_tasks(C)
assert dag.tasks == {A,B,C}
def test__DAG_add_dependency_detect_cycle():
A, B = get_two_tasks()
C = Task('C.py', env='test-env')
dag = DAG()
with pytest.raises(CyclicGraphError):
dag.add_dependencies({
A: C,
B: A,
C: B
})
def test__Dequindre_init():
"""Nothing should break here
"""
A = Task('A.py', 'test-env')
B = Task('B.py', 'test-env')
C = Task('C.py', 'test-env')
dag = DAG()
dag.add_tasks({A, B, C})
dq = Dequindre(dag)
return None
def test__Dequindre_get_task_schedules():
A = Task('A.py', 'test-env')
B = Task('B.py', 'test-env')
C = Task('C.py', 'test-env')
Z = Task('Z.py', 'test-env')
dag = DAG()
dag.add_tasks({A, B, C, Z})
dag.add_dependencies({B: A, C: B})
dq = Dequindre(dag)
priorities = dq.get_task_schedules()
testable = {hash(k): v for k, v in priorities.items()}
assert testable == {hash(A): 1, hash(B): 2, hash(C): 3, hash(Z): 1}
def test__Dequindre_init_exceptions():
"""Raise expected exceptions
"""
A = Task('A.py', 'test-env')
B = Task('B.py', 'test-env')
C = Task('C.py', 'test-env')
dag = DAG()
dag.add_tasks({A, B, C})
with pytest.raises(TypeError):
Dequindre()
return None
def test__common_task_example():
from dequindre import DAG, Dequindre
from dequindre.commons import common_task
with common_task('./tea-tasks/{}', 'python') as TeaTask:
boil_water = TeaTask('boil_water.py')
pour_water = TeaTask('pour_water.py')
prep_infuser = TeaTask('prep_infuser.py')
steep_tea = TeaTask('steep_tea.py')
make_tea = DAG(dependencies={
boil_water: {pour_water},
steep_tea: {boil_water, prep_infuser}
})
## run tasks
dq = Dequindre(make_tea)
dq.run_tasks()
def test__Dequindre_refresh_dag():
A = Task('A.py', 'test-env')
B = Task('B.py', 'test-env')
C = Task('C.py', 'test-env')
dag = DAG()
dag.add_tasks({A, B, C})
dq = Dequindre(dag)
tasks = sorted(list(dq.dag.tasks))
for t in tasks:
dq.dag.remove_task(t)
assert dq.dag.tasks == set()
dq.refresh_dag()
new_tasks = sorted(list(dq.dag.tasks))
for t, nt in zip(tasks, new_tasks):
assert t == nt
def test__DAG_add_dependency_detect_cycle():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, A)
with pytest.raises(CyclicGraphError):
dag.add_dependency(A, B)
def test__DAG_add_dependency():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, A)
assert dag._edges[A] == set([B])
def test__DAG_get_sinks():
A, B = get_two_tasks()
dag = DAG()
dag.add_dependency(B, depends_on=A)
assert dag.get_sinks() is not None
assert dag.get_sinks() == {B,}
def test__Dequindre_repr():
make_tea = Task('make_tea.py', 'test-env')
dag = DAG()
dag.add_task(make_tea)
dq = Dequindre(dag)
assert repr(dq) == "Dequindre(DAG({Task(make_tea.py)}))"
def test__DAG_repr():
make_tea = Task('make_tea.py', 'test-env')
dag = DAG()
dag.add_task(make_tea)
assert repr(dag) == "DAG({Task(make_tea.py)})"