def test__Composer_get_schedules():
A = Task("A.py", "test-exe")
B = Task("B.py", "test-exe")
C = Task("C.py", "test-exe")
Z = Task("Z.py", "test-exe")
dag = DAG()
dag.add_tasks({A, B, C, Z})
dag.add_dependencies({B: A, C: B})
dq = Composer(dag)
priorities = dq.get_schedules()
testable = {}
# build a testable result dict
for k, v in priorities.items():
new_set = set()
if isinstance(v, Task):
testable[k] = set(v)
continue
for vi in v:
new_set.add(hash(vi))
testable[k] = new_set
assert testable == {1: {hash(A), hash(Z)}, 2: {hash(B)}, 3: {hash(C)}}
def test__Composer_get_task_schedules():
A = Task("A.py", "test-exe")
B = Task("B.py", "test-exe")
C = Task("C.py", "test-exe")
Z = Task("Z.py", "test-exe")
dag = DAG()
dag.add_tasks({A, B, C, Z})
dag.add_dependencies({B: A, C: B})
dq = Composer(dag)
priorities = dq.get_task_schedules()
testable = {hash(k): v for k, v in priorities.items()}
assert testable == {hash(A): 1, hash(B): 2, hash(C): 3, hash(Z): 1}
def get_tasks_blocks(compose_yaml: pathlib.Path) -> str:
compose_yaml = pathlib.Path(compose_yaml)
config = parse_config(compose_yaml)
task_map = parse_tasks(config, FLAGS.check_file_presence)
inv_task_map = {v: k for k, v in task_map.items()}
dependency_map = parse_upstream_dependencies(config)
commands = []
composer = Composer.from_yaml(compose_yaml)
for _, tasks in composer.get_schedules().items():
for task in tasks:
comment = comment_task(config,
task,
inv_task_map=inv_task_map,
dependency_map=dependency_map)
cmd = "\n".join([
f"pushd {task.loc.parent} > /dev/null",
f"source activate {task.env}", # `conda activate env` is glitchy
f"python {task.loc.name}",
f"conda deactivate", # `conda deactivate` is not, however...
"popd > /dev/null",
])
# ^^^^^^^^^^^^^^^^
# /path/to/exe \
# /path/to/loc
block = "\n".join([comment, cmd])
commands.append(block)
tasks_block = "\n\n".join(commands)
return tasks_block
def test__readme_example():
## define tasks and environments
pour_water = Task("./tea-tasks/pour_water.py")
boil_water = Task("./tea-tasks/boil_water.py")
prep_infuser = Task("./tea-tasks/prep_infuser.py")
steep_tea = Task("./tea-tasks/steep_tea.py")
## define runtime dependencies
make_tea = DAG(upstream_dependencies={
boil_water: {pour_water},
steep_tea: {boil_water, prep_infuser},
})
## run tasks
dq = Composer(make_tea)
dq.get_schedules()
def test__Composer_repr():
p = pathlib.Path("make_tea.py")
make_tea = Task(p, "test-exe")
dag = DAG()
dag.add_task(make_tea)
dq = Composer(dag)
assert repr(dq) == "".join(
["Composer(DAG({Task(",
p.resolve().as_posix(), ")}))"])
def test__Composer_refresh_dag():
A = Task("A.py", "test-exe")
B = Task("B.py", "test-exe")
C = Task("C.py", "test-exe")
dag = DAG()
dag.add_tasks({A, B, C})
dq = Composer(dag)
tasks = sorted(list(dq.dag.tasks))
for t in tasks:
dq.dag.remove_task(t)
assert dq.dag.tasks == set()
dq.refresh_dag()
new_tasks = sorted(list(dq.dag.tasks))
for t, nt in zip(tasks, new_tasks):
assert t == nt
def test__Composer_init():
"""Nothing should break here
"""
A = Task("A.py", "test-exe")
B = Task("B.py", "test-exe")
C = Task("C.py", "test-exe")
dag = DAG()
dag.add_tasks({A, B, C})
dq = Composer(dag)
return None
def test__Composer_init_exceptions():
"""Raise expected exceptions
"""
A = Task("A.py", "test-exe")
B = Task("B.py", "test-exe")
C = Task("C.py", "test-exe")
dag = DAG()
dag.add_tasks({A, B, C})
with pytest.raises(TypeError):
Composer()
return None
def test__Composer_from_yaml():
Composer.from_yaml(COMPOSE_SMALL)
with pytest.raises(CyclicGraphError):
Composer.from_yaml(COMPOSE_CYCLE)