def test_run_job(n_jobs: int, storage):
task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output())
assert not is_completed(task.input().src_task, storage)
assert not is_completed(task, storage)
run_job(task, storage, n_jobs=n_jobs)
assert is_completed(task.input().src_task, storage)
assert is_completed(task, storage)
def test_run_with_context(method: str, storage):
task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output())
assert not is_completed(task.input().src_task, storage)
assert not is_completed(task, storage)
run_job(task, storage, n_jobs=2, context=get_context(method))
assert is_completed(task.input().src_task, storage)
assert is_completed(task, storage)
def test_with_ephemeral_output_task(task, n_jobs, storage):
run_job(task, storage, n_jobs=2)
assert is_completed(task, storage)
assert task.input().ephemeral
assert not is_completed(
task.input().src_task,
storage), "ephemeral output is already removed automatically"
def test_run_dynamic_task_job(n_jobs, storage):
dynamic = DynamicTask1(
parent=Task1(name="test-dynamic-task", resource_spec=None).output())
# Case - case independently executed
run_job(dynamic.parent.src_task, storage, n_jobs=n_jobs)
task = generate_task(dynamic, storage)
assert isinstance(task, WriteValue)
assert task.value_to_write == "test-dynamic-task"
run_job(task, storage)
assert is_completed(task, storage)
assert is_completed(dynamic, storage)
_flush(storage)
# Case - executed at the same time
assert not is_completed(dynamic,
storage), "confirm initialization of storage"
run_job(dynamic, storage, n_jobs=n_jobs)
assert is_completed(dynamic, storage)
def test_run_with_resources(concurrency, expect, comparator, storage):
items_limited = [
Tagged(resources=["resource-1"], value="op1"),
Tagged(resources=["resource-1"], value="op2"),
Tagged(resources=["resource-1", "resource-2"], value="op3"),
]
main = Tagged(parents=items_limited, value="op4")
t = time.time()
run_job(main, storage, resources={"resource-1": concurrency}, n_jobs=3)
sec = time.time() - t
if comparator == "gt":
assert sec > expect
elif comparator == "lt":
assert sec < expect
else:
raise NotImplementedError
from alexflow.helper import load_output
import json
import tempfile
# Here start the construction of workflow, with using Task and Output objects.
dataset = Dataset()
X, y = dataset.output()
train = Train(X=X, y=y, model_type="Lasso")
predict = Predict(model=train.output(), X=X)
print("Task:")
print(json.dumps(predict.serialize(), indent=4))
# Then here start the execution of the workflow. We can run a workflow over a storage interface. Here we use
# LocalStorage with temporary directory.
with tempfile.TemporaryDirectory() as tempdir:
storage = LocalStorage(base_path=tempdir)
# With run_job, all the dependent task will be resolved as DAG, and executed.
run_job(task=predict, storage=storage)
# Here how you can load the output of the task.
prediction = load_output(predict.output(), storage)
print(f"Generated Prediction:")
print(prediction)
def test_complex_input_task(storage):
task = ComplexInputTask()
run_job(task, storage, n_jobs=1)
assert is_completed(task, storage)