def test_get_output_3(workflow_start_regular, tmp_path): cnt_file = tmp_path / "counter" cnt_file.write_text("0") error_flag = tmp_path / "error" error_flag.touch() @workflow.step def incr(): v = int(cnt_file.read_text()) cnt_file.write_text(str(v + 1)) if error_flag.exists(): raise ValueError() return 10 with pytest.raises(ray.exceptions.RaySystemError): incr.step().run("incr") assert cnt_file.read_text() == "1" with pytest.raises(ray.exceptions.RaySystemError): ray.get(workflow.get_output("incr")) assert cnt_file.read_text() == "1" error_flag.unlink() with pytest.raises(ray.exceptions.RaySystemError): ray.get(workflow.get_output("incr")) assert ray.get(workflow.resume("incr")) == 10
def test_get_named_step_output_finished(workflow_start_regular, tmp_path): @workflow.step def double(v): return 2 * v # Get the result from named step after workflow finished assert 4 == double.options(name="outer").step( double.options(name="inner").step(1)).run("double") assert ray.get(workflow.get_output("double", name="inner")) == 2 assert ray.get(workflow.get_output("double", name="outer")) == 4
def test_get_named_step_duplicate(workflow_start_regular): @workflow.step def f(n, dep): return n inner = f.options(name="f").step(10, None) outer = f.options(name="f").step(20, inner) assert 20 == outer.run("duplicate") # The outer will be checkpointed first. So there is no suffix for the name assert ray.get(workflow.get_output("duplicate", name="f")) == 20 # The inner will be checkpointed after the outer. And there is a duplicate # for the name. suffix _1 is added automatically assert ray.get(workflow.get_output("duplicate", name="f_1")) == 10
def test_get_named_step_output_running(workflow_start_regular, tmp_path): @workflow.step def double(v, lock=None): if lock is not None: with FileLock(lock_path): return 2 * v else: return 2 * v # Get the result from named step after workflow before it's finished lock_path = str(tmp_path / "lock") lock = FileLock(lock_path) lock.acquire() output = double.options(name="outer").step( double.options(name="inner").step(1, lock_path), lock_path).run_async("double-2") inner = workflow.get_output("double-2", name="inner") outer = workflow.get_output("double-2", name="outer") @ray.remote def wait(obj_ref): return ray.get(obj_ref[0]) # Make sure nothing is finished. ready, waiting = ray.wait( [wait.remote([output]), wait.remote([inner]), wait.remote([outer])], timeout=1) assert 0 == len(ready) assert 3 == len(waiting) # Once job finished, we'll be able to get the result. lock.release() assert 4 == ray.get(output) # Here sometimes inner will not be generated when we call # run_async. So there is a race condition here. try: v = ray.get(inner) except Exception: v = None if v is not None: assert 2 == 20 assert 4 == ray.get(outer) inner = workflow.get_output("double-2", name="inner") outer = workflow.get_output("double-2", name="outer") assert 2 == ray.get(inner) assert 4 == ray.get(outer)
def test_get_output_1(workflow_start_regular, tmp_path): @workflow.step def simple(v): return v assert 0 == simple.step(0).run("simple") assert 0 == ray.get(workflow.get_output("simple"))
def test_get_named_step_output_error(workflow_start_regular, tmp_path): @workflow.step def double(v, error): if error: raise Exception() return v + v # Force it to fail for the outer step with pytest.raises(Exception): double.options(name="outer").step( double.options(name="inner").step(1, False), True).run("double") # For the inner step, it should have already been executed. assert 2 == ray.get(workflow.get_output("double", name="inner")) outer = workflow.get_output("double", name="outer") with pytest.raises(Exception): ray.get(outer)
def test_workflow_lifetime_2(call_ray_start): # Case 2: driver terminated proc = run_string_as_driver_nonblocking(driver_script.format(100)) time.sleep(10) proc.kill() time.sleep(1) workflow.init() output = workflow.get_output("driver_terminated") assert ray.get(output) == 20
def test_get_output_2(workflow_start_regular, tmp_path): lock_path = str(tmp_path / "lock") lock = FileLock(lock_path) @workflow.step def simple(v): with FileLock(lock_path): return v lock.acquire() obj = simple.step(0).run_async("simple") obj2 = workflow.get_output("simple") lock.release() assert ray.get([obj, obj2]) == [0, 0]
def test_get_named_step_default(workflow_start_regular, tmp_path): @workflow.step def factorial(n, r=1): if n == 1: return r return factorial.step(n - 1, r * n) import math assert math.factorial(5) == factorial.step(5).run("factorial") for i in range(5): step_name = ("test_basic_workflows_2." "test_get_named_step_default.locals.factorial") if i != 0: step_name += "_" + str(i) # All outputs will be 120 assert math.factorial(5) == ray.get( workflow.get_output("factorial", name=step_name))
def test_workflow_lifetime_1(call_ray_start): # Case 1: driver exits normally run_string_as_driver(driver_script.format(5)) workflow.init() output = workflow.get_output("driver_terminated") assert ray.get(output) == 20