def decide(result: int) -> str: if result < 10: return workflow.continuation(small.bind(result)) elif result < 100: return workflow.continuation(medium.bind(result)) else: return workflow.continuation(large.bind(result))
def handle_errors( car_req_id: str, hotel_req_id: str, flight_req_id: str, final_result: Tuple[Optional[str], Optional[Exception]], ) -> str: result, error = final_result @ray.remote def wait_all(*deps) -> None: pass @ray.remote def cancel(request_id: str) -> None: make_request("cancel", request_id) if error: return workflow.continuation( wait_all.bind( cancel.bind(car_req_id), cancel.bind(hotel_req_id), cancel.bind(flight_req_id), )) else: return result
def recursive(n): if n <= 0: with FileLock(lock_path): return 42 return workflow.continuation( recursive.options(**workflow.options(name=str(n - 1))).bind(n - 1) )
def exponential_fail(k, n): if n > 0: if n < 3: raise Exception("Failed intentionally") return workflow.continuation( exponential_fail.options(name=f"step_{n}").bind(k * 2, n - 1)) return k
def exit_handler(res: Tuple[Optional[str], Optional[Exception]]) -> None: result, error = res email = send_email.bind(f"Raw result: {result}, {error}") if error: handler = cry.bind(error) else: handler = celebrate.bind(result) return workflow.continuation(wait_all.bind(handler, email))
def complex(x1): x2 = source1.bind() v = join.bind(x1, x2) y = append1.bind(x1) y = the_failed_step.bind(y) z = append2.bind(x2) u = join.bind(y, z) return workflow.continuation(join.bind(u, v))
def flip_coin() -> str: import random @ray.remote def decide(heads: bool) -> str: return workflow.continuation( handle_heads.bind() if heads else handle_tails.bind()) return workflow.continuation(decide.bind(random.random() > 0.5))
def checkpoint_dag(checkpoint): x = utils.update_workflow_options( large_input, name="large_input", checkpoint=checkpoint ).bind() y = utils.update_workflow_options( identity, name="identity", checkpoint=checkpoint ).bind(x) return workflow.continuation( utils.update_workflow_options(average, name="average").bind(y) )
def inplace_test(): from ray.worker import global_worker worker_id = global_worker.worker_id x = check_and_update.options(**workflow.options(allow_inplace=True)).bind( "@", worker_id) y = check_and_update.bind(x, worker_id) z = check_and_update.options(**workflow.options(allow_inplace=True)).bind( y, worker_id) return workflow.continuation(z)
def tail_recursion(n): import inspect # check if the stack is growing assert len(inspect.stack(0)) < 20 if n <= 0: return "ok" return workflow.continuation( tail_recursion.options(**workflow.options( allow_inplace=True)).bind(n - 1))
def compute_large_fib(M: int, n: int = 1, fib: int = 1): next_fib = requests.post("https://nemo.api.stdlib.com/[email protected]/", data={ "nth": n }).json() if next_fib > M: return fib else: return workflow.continuation(compute_large_fib.bind( M, n + 1, next_fib))
def book_all(car_req_id: str, hotel_req_id: str, flight_req_id: str) -> str: car_res_id = book_car.bind(car_req_id) hotel_res_id = book_hotel.bind(hotel_req_id, car_res_id) flight_res_id = book_flight.bind(hotel_req_id, hotel_res_id) @ray.remote def concat(*ids: List[str]) -> str: return ", ".join(ids) return workflow.continuation(concat.bind(car_res_id, hotel_res_id, flight_res_id))
def handle_result(res: Tuple[Optional[str], Optional[Exception]]) -> str: result, error = res if result: return res elif num_retries <= 0: raise error else: print("Retrying exception after delay", error) time.sleep(delay_s) return workflow.continuation( custom_retry_strategy.bind(func, num_retries - 1, delay_s))
def exp_remote(k, n, worker_id=None): from ray.worker import global_worker _worker_id = global_worker.worker_id if worker_id is not None: # sub-workflows running in another worker assert _worker_id != worker_id worker_id = _worker_id if n == 0: return k return workflow.continuation(exp_remote.bind(2 * k, n - 1, worker_id))
def test_workflow_continuation(workflow_start_regular_shared): """Test unified behavior of returning continuation inside workflow and default Ray execution engine.""" @ray.remote def h(a, b): return a + b @ray.remote def g(x): return workflow.continuation(h.bind(42, x)) @ray.remote def f(): return workflow.continuation(g.bind(1)) with pytest.raises(TypeError): workflow.continuation(f.remote()) dag = f.bind() assert ray.get(dag.execute()) == 43 assert workflow.create(dag).run() == 43
def exp_inplace(k, n, worker_id=None): from ray.worker import global_worker _worker_id = global_worker.worker_id if worker_id is not None: # sub-workflows running inplace assert _worker_id == worker_id worker_id = _worker_id if n == 0: return k return workflow.continuation( exp_inplace.options(**workflow.options(allow_inplace=True)).bind( 2 * k, n - 1, worker_id))
def compute_large_fib(M: int, n: int = 1, fib: int = 1): try: next_fib = requests.post( "https://nemo.api.stdlib.com/[email protected]/", data={ "nth": n }).json() assert isinstance(next_fib, int) except AssertionError: # TODO(suquark): The web service would fail sometimes. This is a workaround. next_fib = fibonacci(n) if next_fib > M: return fib else: return workflow.continuation(compute_large_fib.bind( M, n + 1, next_fib))
def custom_retry_strategy(func: Any, num_retries: int, delay_s: int) -> str: import time @ray.remote def handle_result(res: Tuple[Optional[str], Optional[Exception]]) -> str: result, error = res if result: return res elif num_retries <= 0: raise error else: print("Retrying exception after delay", error) time.sleep(delay_s) return workflow.continuation( custom_retry_strategy.bind(func, num_retries - 1, delay_s)) res = func.options(**workflow.options(catch_exceptions=True)).bind() return workflow.continuation(handle_result.bind(res))
def checkpoint_dag(checkpoint): @ray.remote def large_input(): return np.arange(2**24) @ray.remote def identity(x): return x @ray.remote def average(x): return np.mean(x) x = large_input.options( **workflow.options(name="large_input", checkpoint=checkpoint)).bind() y = identity.options( **workflow.options(name="identity", checkpoint=checkpoint)).bind(x) return workflow.continuation( average.options(**workflow.options(name="average")).bind(y))
def checkpoint_dag(checkpoint): @ray.remote def large_input(): return np.arange(SIZE) @ray.remote def identity(x): if not utils.check_global_mark(): import os os.kill(os.getpid(), 9) return x @ray.remote def average(x): return np.mean(x) x = large_input.options(**workflow.options(checkpoint=checkpoint)).bind() y = identity.options(**workflow.options(checkpoint=checkpoint)).bind(x) return workflow.continuation(average.bind(y))
def iterate(array: List[str], result: str, i: int) -> str: if i >= len(array): return result return workflow.continuation(iterate.bind(array, result + array[i], i + 1))
def hello(name: str) -> str: return workflow.continuation(format_name.bind(name))
def scan(x0: str, x1: str, x2: str): x0 = sha1((x0 + x2).encode()).hexdigest() x1 = sha1((x1 + x2).encode()).hexdigest() x2 = sha1((x0 + x1 + x2).encode()).hexdigest() y0, y1, y2 = pass_1.bind(x0, x1), pass_2.bind(x1, x2), pass_3.bind(x2, x0) return workflow.continuation(merge.bind(y0, y1, y2))
def pass_3(x: str, y: str): if sha1((x + y + "_3").encode()).hexdigest() > x: return sha1((x + y + "3").encode()).hexdigest() return workflow.continuation(pass_2.bind(x, y))
def outer(): time.sleep(2) return workflow.continuation(inner.bind())
def fork_join(): x = source1.bind() y = append1.bind(x) y = identity.bind(y) z = append2.bind(x) return workflow.continuation(join.bind(y, z))
def factorial(n): if n == 1: return 1 else: return workflow.continuation(mul.bind(n, factorial.bind(n - 1)))
def handle_tails() -> str: print("It was tails, retrying") return workflow.continuation(flip_coin.bind())
def checkpoint_dag2(checkpoint): x = utils.update_workflow_options(large_input, checkpoint=checkpoint).bind() y = utils.update_workflow_options(identity2, checkpoint=checkpoint).bind(x) return workflow.continuation(average.bind(y))
def decide(heads: bool) -> str: if heads: return workflow.continuation(handle_heads.bind()) else: return workflow.continuation(handle_tails.bind())