def test_recovery_simple(workflow_start_regular):
utils.unset_global_mark()
workflow_id = "test_recovery_simple"
with pytest.raises(RaySystemError):
# internally we get WorkerCrashedError
simple.step("x").run(workflow_id=workflow_id)
utils.set_global_mark()
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
def test_recovery_complex(workflow_start_regular):
utils.unset_global_mark()
workflow_id = "test_recovery_complex"
with pytest.raises(RaySystemError):
# internally we get WorkerCrashedError
complex.step("x").run(workflow_id=workflow_id)
utils.set_global_mark()
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
def test_recovery_simple():
ray.init()
utils.unset_global_mark()
workflow_id = "test_recovery_simple"
with pytest.raises(ObjectLostError):
# internally we get WorkerCrashedError
output = workflow.run(simple.step("x"), workflow_id=workflow_id)
ray.get(output)
utils.set_global_mark()
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
assert ray.get(output) == "foo(x[append1])[append2]"
ray.shutdown()
def test_recovery_complex():
ray.init()
utils.unset_global_mark()
workflow_id = "test_recovery_complex"
with pytest.raises(RayTaskError):
# internally we get WorkerCrashedError
output = workflow.run(complex.step("x"), workflow_id=workflow_id)
ray.get(output)
utils.set_global_mark()
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
utils.unset_global_mark()
# resume from workflow output checkpoint
output = workflow.resume(workflow_id)
r = "join(join(foo(x[append1]), [source1][append2]), join(x, [source1]))"
assert ray.get(output) == r
ray.shutdown()