def input(self):
return {
"value": Task1(name="task1").output(),
"array": [Task1(name="task2").output()],
"dict": {
"task3": Task1(name="task3").output(),
},
}
def test_run_dynamic_task_job():
dynamic = DynamicTask1(parent=Task1(name="test-dynamic-task").output())
# Case - case independently executed
with tempfile.TemporaryDirectory() as dirname:
storage = LocalStorage(dirname)
run_job(dynamic.parent.src_task, storage)
task = generate_task(dynamic, storage)
assert isinstance(task, WriteValue)
assert task.value_to_write == "test-dynamic-task"
run_job(task, storage)
assert is_completed(task, storage)
assert is_completed(dynamic, storage)
assert not is_completed(dynamic, storage)
# Case - executed at the same time
with tempfile.TemporaryDirectory() as dirname:
run_job(dynamic, storage)
assert is_completed(dynamic, storage)
def test_run_dynamic_task_job(n_jobs, storage):
dynamic = DynamicTask1(
parent=Task1(name="test-dynamic-task", resource_spec=None).output())
# Case - case independently executed
run_job(dynamic.parent.src_task, storage, n_jobs=n_jobs)
task = generate_task(dynamic, storage)
assert isinstance(task, WriteValue)
assert task.value_to_write == "test-dynamic-task"
run_job(task, storage)
assert is_completed(task, storage)
assert is_completed(dynamic, storage)
_flush(storage)
# Case - executed at the same time
assert not is_completed(dynamic,
storage), "confirm initialization of storage"
run_job(dynamic, storage, n_jobs=n_jobs)
assert is_completed(dynamic, storage)
def test_run_job(n_jobs: int, storage):
task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output())
assert not is_completed(task.input().src_task, storage)
assert not is_completed(task, storage)
run_job(task, storage, n_jobs=n_jobs)
assert is_completed(task.input().src_task, storage)
assert is_completed(task, storage)
def test_run_with_context(method: str, storage):
task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output())
assert not is_completed(task.input().src_task, storage)
assert not is_completed(task, storage)
run_job(task, storage, n_jobs=2, context=get_context(method))
assert is_completed(task.input().src_task, storage)
assert is_completed(task, storage)
def test_run_job():
task = Task2(parent=Task1().output())
with tempfile.TemporaryDirectory() as dirname:
storage = LocalStorage(dirname)
assert not is_completed(task.input().src_task, storage)
assert not is_completed(task, storage)
run_job(task, storage)
assert is_completed(task.input().src_task, storage)
assert is_completed(task, storage)
def test_if_reference_manager_can_handle_mixed_ephemeral_condition():
"""
For the case 1 output is represented as both ephemeral and none-ephemeral, the output
object is finally none-ephemeral and should not be deleted.
"""
storage = MagicMock()
base = Task1()
variant1 = Task2(parent=base.output(), name="variant1")
variant2 = Task2(parent=base.output().as_ephemeral(), name="variant2")
tasks = [variant1, variant2]
manager = ReferenceManager({task.task_id: task for task in tasks}, storage)
assert manager._ephemeral_map[base.output().key] is False
manager.remove(base)
manager.remove(variant1)
manager.remove(variant2)
storage.remove.assert_not_called()
def test_complex_input_task(storage):
task = ComplexInputTask()
run_job(task, storage, n_jobs=1)
assert is_completed(task, storage)
@pytest.mark.parametrize("n_jobs", [1, 2])
@pytest.mark.parametrize(
"task",
[
Task2(parent=Task2(
parent=Task1().output().as_ephemeral()).output().as_ephemeral()),
DynamicTask1(parent=Task1(name="test-dynamic-task",
resource_spec=None).output().as_ephemeral()),
],
)
def test_with_ephemeral_output_task(task, n_jobs, storage):
run_job(task, storage, n_jobs=2)
assert is_completed(task, storage)
assert task.input().ephemeral
assert not is_completed(
task.input().src_task,
storage), "ephemeral output is already removed automatically"