def test_object_deref():
ray.init()
output = workflow.run(
deref_check.step(
ray.put(42), nested_ref.remote(), [nested_ref.remote()],
nested_workflow.step(10), [nested_workflow.step(9)], [{
"output": nested_workflow.step(7)
}]), )
assert ray.get(output)
x = empty_list.step()
output = workflow.run(deref_shared.step(x, x))
assert ray.get(output)
# test we are forbidden from directly passing workflow to Ray.
x = empty_list.step()
with pytest.raises(ValueError):
ray.put(x)
with pytest.raises(ValueError):
ray.get(receive_workflow.remote(x))
with pytest.raises(ValueError):
ray.get(return_workflow.remote())
# test return object ref
obj = return_data.step()
arr: np.ndarray = ray.get(workflow.run(receive_data.step(obj)))
assert np.array_equal(arr, np.ones(4096))
ray.shutdown()
def test_run_or_resume_during_running():
ray.init(namespace="workflow")
output = workflow.run(
simple_sequential.step(), workflow_id="running_workflow")
with pytest.raises(ValueError):
workflow.run(simple_sequential.step(), workflow_id="running_workflow")
with pytest.raises(ValueError):
workflow.resume(workflow_id="running_workflow")
assert ray.get(output) == "[source1][append1][append2]"
ray.shutdown()
def test_recursion_factorial():
import ray
ray.init()
outputs = workflow.run(recursion_factorial.step(10))
assert ray.get(outputs) == 3628800
ray.shutdown()
def test_objectref_inputs_exception():
ray.init()
with pytest.raises(ValueError):
output = workflow.run(receive_data.step(ray.put([42])))
assert ray.get(output)
ray.shutdown()
def test_variable_mutable():
import ray
ray.init(namespace="workflow")
outputs = workflow.run(variable_mutable.step())
assert ray.get(outputs) == []
ray.shutdown()
def test_shortcut():
ray.init()
output = workflow.run(recursive_chain.step(0), workflow_id="shortcut")
assert ray.get(output) == 100
# the shortcut points to the step with output checkpoint
store = workflow_storage.WorkflowStorage("shortcut")
step_id = store.get_entrypoint_step_id()
assert store.inspect_step(step_id).output_object_valid
ray.shutdown()
def test_async_execution():
ray.init()
start = time.time()
output = workflow.run(blocking.step())
duration = time.time() - start
assert duration < 5 # workflow.run is not blocked
assert ray.get(output) == 314
ray.shutdown()
def test_basic_workflows():
ray.init()
output = workflow.run(simple_sequential.step())
assert ray.get(output) == "[source1][append1][append2]"
output = workflow.run(simple_sequential_with_input.step("start:"))
assert ray.get(output) == "start:[append1][append2]"
output = workflow.run(loop_sequential.step(3))
assert ray.get(output) == "[source1]" + "[append1]" * 3 + "[append2]"
output = workflow.run(nested.step("nested:"))
assert ray.get(output) == "nested:~[nested]~[append1][append2]"
output = workflow.run(fork_join.step())
assert ray.get(output) == "join([source1][append1], [source1][append2])"
ray.shutdown()
def test_simple_large_intermediate():
import ray
ray.init(namespace="workflow")
start = time.time()
outputs = workflow.run(simple_large_intermediate.step())
outputs = ray.get(outputs)
print(f"duration = {time.time() - start}")
assert np.isclose(outputs, 8388607.5)
ray.shutdown()
def test_objectref_inputs():
ray.init()
output = workflow.run(
deref_check.step(
ray.put(42), nested_ref.remote(), [nested_ref.remote()],
nested_workflow.step(10), [nested_workflow.step(9)], [{
"output": nested_workflow.step(7)
}]))
assert ray.get(output)
ray.shutdown()
def test_object_deref():
ray.init(namespace="workflow")
x = empty_list.step()
output = workflow.run(deref_shared.step(x, x))
assert ray.get(output)
# test we are forbidden from directly passing workflow to Ray.
x = empty_list.step()
with pytest.raises(ValueError):
ray.put(x)
with pytest.raises(ValueError):
ray.get(receive_workflow.remote(x))
with pytest.raises(ValueError):
ray.get(return_workflow.remote())
# test return object ref
obj = return_data.step()
arr: np.ndarray = ray.get(workflow.run(receive_data.step(obj)))
assert np.array_equal(arr, np.ones(4096))
ray.shutdown()
def test_recovery_simple():
ray.init()
utils.unset_global_mark()
workflow_id = "test_recovery_simple"
with pytest.raises(RaySystemError):
# internally we get WorkerCrashedError
output = workflow.run(simple.step("x"), workflow_id=workflow_id)
ray.get(output)
utils.set_global_mark()
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
ray.shutdown()
def test_recovery_complex():
ray.init()
utils.unset_global_mark()
workflow_id = "test_recovery_complex"
with pytest.raises(RaySystemError):
# internally we get WorkerCrashedError
output = workflow.run(complex.step("x"), workflow_id=workflow_id)
ray.get(output)
utils.set_global_mark()
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
ray.shutdown()
import sys
import time
import ray
from ray.experimental import workflow
@workflow.step
def foo(x):
print("Executing", x)
time.sleep(1)
if x < 20:
return foo.step(x + 1)
else:
return 20
if __name__ == "__main__":
sleep_duration = float(sys.argv[1])
ray.init(address="auto", namespace="workflow")
wf = workflow.run(foo.step(0), workflow_id="driver_terminated")
time.sleep(sleep_duration)
import time
import ray
from ray.experimental import workflow
@workflow.step
def foo(x):
print("Executing", x)
time.sleep(1)
if x < 20:
return foo.step(x + 1)
else:
return 20
if __name__ == "__main__":
ray.init(address="auto", namespace="workflow")
wf = workflow.run(foo.step(0), workflow_id="cluster_failure")
assert ray.get(wf) == 20
