def decide(result: int) -> str:
if result < 10:
return workflow.continuation(small.bind(result))
elif result < 100:
return workflow.continuation(medium.bind(result))
else:
return workflow.continuation(large.bind(result))
def handle_errors(
car_req_id: str,
hotel_req_id: str,
flight_req_id: str,
final_result: Tuple[Optional[str], Optional[Exception]],
) -> str:
result, error = final_result
@ray.remote
def wait_all(*deps) -> None:
pass
@ray.remote
def cancel(request_id: str) -> None:
make_request("cancel", request_id)
if error:
return workflow.continuation(
wait_all.bind(
cancel.bind(car_req_id),
cancel.bind(hotel_req_id),
cancel.bind(flight_req_id),
))
else:
return result
def recursive(n):
if n <= 0:
with FileLock(lock_path):
return 42
return workflow.continuation(
recursive.options(**workflow.options(name=str(n - 1))).bind(n - 1)
)
def exponential_fail(k, n):
if n > 0:
if n < 3:
raise Exception("Failed intentionally")
return workflow.continuation(
exponential_fail.options(name=f"step_{n}").bind(k * 2, n - 1))
return k
def exit_handler(res: Tuple[Optional[str], Optional[Exception]]) -> None:
result, error = res
email = send_email.bind(f"Raw result: {result}, {error}")
if error:
handler = cry.bind(error)
else:
handler = celebrate.bind(result)
return workflow.continuation(wait_all.bind(handler, email))
def complex(x1):
x2 = source1.bind()
v = join.bind(x1, x2)
y = append1.bind(x1)
y = the_failed_step.bind(y)
z = append2.bind(x2)
u = join.bind(y, z)
return workflow.continuation(join.bind(u, v))
def flip_coin() -> str:
import random
@ray.remote
def decide(heads: bool) -> str:
return workflow.continuation(
handle_heads.bind() if heads else handle_tails.bind())
return workflow.continuation(decide.bind(random.random() > 0.5))
def checkpoint_dag(checkpoint):
x = utils.update_workflow_options(
large_input, name="large_input", checkpoint=checkpoint
).bind()
y = utils.update_workflow_options(
identity, name="identity", checkpoint=checkpoint
).bind(x)
return workflow.continuation(
utils.update_workflow_options(average, name="average").bind(y)
)
def inplace_test():
from ray.worker import global_worker
worker_id = global_worker.worker_id
x = check_and_update.options(**workflow.options(allow_inplace=True)).bind(
"@", worker_id)
y = check_and_update.bind(x, worker_id)
z = check_and_update.options(**workflow.options(allow_inplace=True)).bind(
y, worker_id)
return workflow.continuation(z)
def tail_recursion(n):
import inspect
# check if the stack is growing
assert len(inspect.stack(0)) < 20
if n <= 0:
return "ok"
return workflow.continuation(
tail_recursion.options(**workflow.options(
allow_inplace=True)).bind(n - 1))
def compute_large_fib(M: int, n: int = 1, fib: int = 1):
next_fib = requests.post("https://nemo.api.stdlib.com/[email protected]/",
data={
"nth": n
}).json()
if next_fib > M:
return fib
else:
return workflow.continuation(compute_large_fib.bind(
M, n + 1, next_fib))
def book_all(car_req_id: str, hotel_req_id: str, flight_req_id: str) -> str:
car_res_id = book_car.bind(car_req_id)
hotel_res_id = book_hotel.bind(hotel_req_id, car_res_id)
flight_res_id = book_flight.bind(hotel_req_id, hotel_res_id)
@ray.remote
def concat(*ids: List[str]) -> str:
return ", ".join(ids)
return workflow.continuation(concat.bind(car_res_id, hotel_res_id, flight_res_id))
def handle_result(res: Tuple[Optional[str], Optional[Exception]]) -> str:
result, error = res
if result:
return res
elif num_retries <= 0:
raise error
else:
print("Retrying exception after delay", error)
time.sleep(delay_s)
return workflow.continuation(
custom_retry_strategy.bind(func, num_retries - 1, delay_s))
def exp_remote(k, n, worker_id=None):
from ray.worker import global_worker
_worker_id = global_worker.worker_id
if worker_id is not None:
# sub-workflows running in another worker
assert _worker_id != worker_id
worker_id = _worker_id
if n == 0:
return k
return workflow.continuation(exp_remote.bind(2 * k, n - 1, worker_id))
def test_workflow_continuation(workflow_start_regular_shared):
"""Test unified behavior of returning continuation inside
workflow and default Ray execution engine."""
@ray.remote
def h(a, b):
return a + b
@ray.remote
def g(x):
return workflow.continuation(h.bind(42, x))
@ray.remote
def f():
return workflow.continuation(g.bind(1))
with pytest.raises(TypeError):
workflow.continuation(f.remote())
dag = f.bind()
assert ray.get(dag.execute()) == 43
assert workflow.create(dag).run() == 43
def exp_inplace(k, n, worker_id=None):
from ray.worker import global_worker
_worker_id = global_worker.worker_id
if worker_id is not None:
# sub-workflows running inplace
assert _worker_id == worker_id
worker_id = _worker_id
if n == 0:
return k
return workflow.continuation(
exp_inplace.options(**workflow.options(allow_inplace=True)).bind(
2 * k, n - 1, worker_id))
def compute_large_fib(M: int, n: int = 1, fib: int = 1):
try:
next_fib = requests.post(
"https://nemo.api.stdlib.com/[email protected]/", data={
"nth": n
}).json()
assert isinstance(next_fib, int)
except AssertionError:
# TODO(suquark): The web service would fail sometimes. This is a workaround.
next_fib = fibonacci(n)
if next_fib > M:
return fib
else:
return workflow.continuation(compute_large_fib.bind(
M, n + 1, next_fib))
def custom_retry_strategy(func: Any, num_retries: int, delay_s: int) -> str:
import time
@ray.remote
def handle_result(res: Tuple[Optional[str], Optional[Exception]]) -> str:
result, error = res
if result:
return res
elif num_retries <= 0:
raise error
else:
print("Retrying exception after delay", error)
time.sleep(delay_s)
return workflow.continuation(
custom_retry_strategy.bind(func, num_retries - 1, delay_s))
res = func.options(**workflow.options(catch_exceptions=True)).bind()
return workflow.continuation(handle_result.bind(res))
def checkpoint_dag(checkpoint):
@ray.remote
def large_input():
return np.arange(2**24)
@ray.remote
def identity(x):
return x
@ray.remote
def average(x):
return np.mean(x)
x = large_input.options(
**workflow.options(name="large_input", checkpoint=checkpoint)).bind()
y = identity.options(
**workflow.options(name="identity", checkpoint=checkpoint)).bind(x)
return workflow.continuation(
average.options(**workflow.options(name="average")).bind(y))
def checkpoint_dag(checkpoint):
@ray.remote
def large_input():
return np.arange(SIZE)
@ray.remote
def identity(x):
if not utils.check_global_mark():
import os
os.kill(os.getpid(), 9)
return x
@ray.remote
def average(x):
return np.mean(x)
x = large_input.options(**workflow.options(checkpoint=checkpoint)).bind()
y = identity.options(**workflow.options(checkpoint=checkpoint)).bind(x)
return workflow.continuation(average.bind(y))
def iterate(array: List[str], result: str, i: int) -> str:
if i >= len(array):
return result
return workflow.continuation(iterate.bind(array, result + array[i], i + 1))
def hello(name: str) -> str:
return workflow.continuation(format_name.bind(name))
def scan(x0: str, x1: str, x2: str):
x0 = sha1((x0 + x2).encode()).hexdigest()
x1 = sha1((x1 + x2).encode()).hexdigest()
x2 = sha1((x0 + x1 + x2).encode()).hexdigest()
y0, y1, y2 = pass_1.bind(x0, x1), pass_2.bind(x1, x2), pass_3.bind(x2, x0)
return workflow.continuation(merge.bind(y0, y1, y2))
def pass_3(x: str, y: str):
if sha1((x + y + "_3").encode()).hexdigest() > x:
return sha1((x + y + "3").encode()).hexdigest()
return workflow.continuation(pass_2.bind(x, y))
def outer():
time.sleep(2)
return workflow.continuation(inner.bind())
def fork_join():
x = source1.bind()
y = append1.bind(x)
y = identity.bind(y)
z = append2.bind(x)
return workflow.continuation(join.bind(y, z))
def factorial(n):
if n == 1:
return 1
else:
return workflow.continuation(mul.bind(n, factorial.bind(n - 1)))
def handle_tails() -> str:
print("It was tails, retrying")
return workflow.continuation(flip_coin.bind())
def checkpoint_dag2(checkpoint):
x = utils.update_workflow_options(large_input, checkpoint=checkpoint).bind()
y = utils.update_workflow_options(identity2, checkpoint=checkpoint).bind(x)
return workflow.continuation(average.bind(y))
def decide(heads: bool) -> str:
if heads:
return workflow.continuation(handle_heads.bind())
else:
return workflow.continuation(handle_tails.bind())
