def test_object_deref(): ray.init() output = workflow.run( deref_check.step( ray.put(42), nested_ref.remote(), [nested_ref.remote()], nested_workflow.step(10), [nested_workflow.step(9)], [{ "output": nested_workflow.step(7) }]), ) assert ray.get(output) x = empty_list.step() output = workflow.run(deref_shared.step(x, x)) assert ray.get(output) # test we are forbidden from directly passing workflow to Ray. x = empty_list.step() with pytest.raises(ValueError): ray.put(x) with pytest.raises(ValueError): ray.get(receive_workflow.remote(x)) with pytest.raises(ValueError): ray.get(return_workflow.remote()) # test return object ref obj = return_data.step() arr: np.ndarray = ray.get(workflow.run(receive_data.step(obj))) assert np.array_equal(arr, np.ones(4096)) ray.shutdown()
def test_run_or_resume_during_running(): ray.init(namespace="workflow") output = workflow.run( simple_sequential.step(), workflow_id="running_workflow") with pytest.raises(ValueError): workflow.run(simple_sequential.step(), workflow_id="running_workflow") with pytest.raises(ValueError): workflow.resume(workflow_id="running_workflow") assert ray.get(output) == "[source1][append1][append2]" ray.shutdown()
def test_recursion_factorial(): import ray ray.init() outputs = workflow.run(recursion_factorial.step(10)) assert ray.get(outputs) == 3628800 ray.shutdown()
def test_objectref_inputs_exception(): ray.init() with pytest.raises(ValueError): output = workflow.run(receive_data.step(ray.put([42]))) assert ray.get(output) ray.shutdown()
def test_variable_mutable(): import ray ray.init(namespace="workflow") outputs = workflow.run(variable_mutable.step()) assert ray.get(outputs) == [] ray.shutdown()
def test_shortcut(): ray.init() output = workflow.run(recursive_chain.step(0), workflow_id="shortcut") assert ray.get(output) == 100 # the shortcut points to the step with output checkpoint store = workflow_storage.WorkflowStorage("shortcut") step_id = store.get_entrypoint_step_id() assert store.inspect_step(step_id).output_object_valid ray.shutdown()
def test_async_execution(): ray.init() start = time.time() output = workflow.run(blocking.step()) duration = time.time() - start assert duration < 5 # workflow.run is not blocked assert ray.get(output) == 314 ray.shutdown()
def test_basic_workflows(): ray.init() output = workflow.run(simple_sequential.step()) assert ray.get(output) == "[source1][append1][append2]" output = workflow.run(simple_sequential_with_input.step("start:")) assert ray.get(output) == "start:[append1][append2]" output = workflow.run(loop_sequential.step(3)) assert ray.get(output) == "[source1]" + "[append1]" * 3 + "[append2]" output = workflow.run(nested.step("nested:")) assert ray.get(output) == "nested:~[nested]~[append1][append2]" output = workflow.run(fork_join.step()) assert ray.get(output) == "join([source1][append1], [source1][append2])" ray.shutdown()
def test_simple_large_intermediate(): import ray ray.init(namespace="workflow") start = time.time() outputs = workflow.run(simple_large_intermediate.step()) outputs = ray.get(outputs) print(f"duration = {time.time() - start}") assert np.isclose(outputs, 8388607.5) ray.shutdown()
def test_objectref_inputs(): ray.init() output = workflow.run( deref_check.step( ray.put(42), nested_ref.remote(), [nested_ref.remote()], nested_workflow.step(10), [nested_workflow.step(9)], [{ "output": nested_workflow.step(7) }])) assert ray.get(output) ray.shutdown()
def test_object_deref(): ray.init(namespace="workflow") x = empty_list.step() output = workflow.run(deref_shared.step(x, x)) assert ray.get(output) # test we are forbidden from directly passing workflow to Ray. x = empty_list.step() with pytest.raises(ValueError): ray.put(x) with pytest.raises(ValueError): ray.get(receive_workflow.remote(x)) with pytest.raises(ValueError): ray.get(return_workflow.remote()) # test return object ref obj = return_data.step() arr: np.ndarray = ray.get(workflow.run(receive_data.step(obj))) assert np.array_equal(arr, np.ones(4096)) ray.shutdown()
def test_recovery_simple(): ray.init() utils.unset_global_mark() workflow_id = "test_recovery_simple" with pytest.raises(RaySystemError): # internally we get WorkerCrashedError output = workflow.run(simple.step("x"), workflow_id=workflow_id) ray.get(output) utils.set_global_mark() output = workflow.resume(workflow_id) assert ray.get(output) == "foo(x[append1])[append2]" utils.unset_global_mark() # resume from workflow output checkpoint output = workflow.resume(workflow_id) assert ray.get(output) == "foo(x[append1])[append2]" ray.shutdown()
def test_recovery_complex(): ray.init() utils.unset_global_mark() workflow_id = "test_recovery_complex" with pytest.raises(RaySystemError): # internally we get WorkerCrashedError output = workflow.run(complex.step("x"), workflow_id=workflow_id) ray.get(output) utils.set_global_mark() output = workflow.resume(workflow_id) r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))" assert ray.get(output) == r utils.unset_global_mark() # resume from workflow output checkpoint output = workflow.resume(workflow_id) r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))" assert ray.get(output) == r ray.shutdown()
import sys import time import ray from ray.experimental import workflow @workflow.step def foo(x): print("Executing", x) time.sleep(1) if x < 20: return foo.step(x + 1) else: return 20 if __name__ == "__main__": sleep_duration = float(sys.argv[1]) ray.init(address="auto", namespace="workflow") wf = workflow.run(foo.step(0), workflow_id="driver_terminated") time.sleep(sleep_duration)
import time import ray from ray.experimental import workflow @workflow.step def foo(x): print("Executing", x) time.sleep(1) if x < 20: return foo.step(x + 1) else: return 20 if __name__ == "__main__": ray.init(address="auto", namespace="workflow") wf = workflow.run(foo.step(0), workflow_id="cluster_failure") assert ray.get(wf) == 20