def test_run_job(n_jobs: int, storage): task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output()) assert not is_completed(task.input().src_task, storage) assert not is_completed(task, storage) run_job(task, storage, n_jobs=n_jobs) assert is_completed(task.input().src_task, storage) assert is_completed(task, storage)
def test_run_with_context(method: str, storage): task = Task2(resource_spec=None, parent=Task1(resource_spec=None).output()) assert not is_completed(task.input().src_task, storage) assert not is_completed(task, storage) run_job(task, storage, n_jobs=2, context=get_context(method)) assert is_completed(task.input().src_task, storage) assert is_completed(task, storage)
def test_with_ephemeral_output_task(task, n_jobs, storage): run_job(task, storage, n_jobs=2) assert is_completed(task, storage) assert task.input().ephemeral assert not is_completed( task.input().src_task, storage), "ephemeral output is already removed automatically"
def test_run_dynamic_task_job(n_jobs, storage): dynamic = DynamicTask1( parent=Task1(name="test-dynamic-task", resource_spec=None).output()) # Case - case independently executed run_job(dynamic.parent.src_task, storage, n_jobs=n_jobs) task = generate_task(dynamic, storage) assert isinstance(task, WriteValue) assert task.value_to_write == "test-dynamic-task" run_job(task, storage) assert is_completed(task, storage) assert is_completed(dynamic, storage) _flush(storage) # Case - executed at the same time assert not is_completed(dynamic, storage), "confirm initialization of storage" run_job(dynamic, storage, n_jobs=n_jobs) assert is_completed(dynamic, storage)
def test_run_with_resources(concurrency, expect, comparator, storage): items_limited = [ Tagged(resources=["resource-1"], value="op1"), Tagged(resources=["resource-1"], value="op2"), Tagged(resources=["resource-1", "resource-2"], value="op3"), ] main = Tagged(parents=items_limited, value="op4") t = time.time() run_job(main, storage, resources={"resource-1": concurrency}, n_jobs=3) sec = time.time() - t if comparator == "gt": assert sec > expect elif comparator == "lt": assert sec < expect else: raise NotImplementedError
from alexflow.helper import load_output import json import tempfile # Here start the construction of workflow, with using Task and Output objects. dataset = Dataset() X, y = dataset.output() train = Train(X=X, y=y, model_type="Lasso") predict = Predict(model=train.output(), X=X) print("Task:") print(json.dumps(predict.serialize(), indent=4)) # Then here start the execution of the workflow. We can run a workflow over a storage interface. Here we use # LocalStorage with temporary directory. with tempfile.TemporaryDirectory() as tempdir: storage = LocalStorage(base_path=tempdir) # With run_job, all the dependent task will be resolved as DAG, and executed. run_job(task=predict, storage=storage) # Here how you can load the output of the task. prediction = load_output(predict.output(), storage) print(f"Generated Prediction:") print(prediction)
def test_complex_input_task(storage): task = ComplexInputTask() run_job(task, storage, n_jobs=1) assert is_completed(task, storage)