def test_map_gh23107(init): sleep_time = 40 # Reference - https://github.com/ray-project/ray/issues/23107 @ray.remote class DummyActor: async def identity(self, s): if s == 6: await asyncio.sleep(sleep_time) return s, time.time() def func(a, v): return a.identity.remote(v) map_values = [1, 2, 3, 4, 5] pool_map = ActorPool([DummyActor.remote() for i in range(2)]) pool_map.submit(func, 6) start_time = time.time() gen = pool_map.map(func, map_values) assert all(elem[0] in [1, 2, 3, 4, 5] for elem in list(gen)) assert all( abs(elem[1] - start_time) < sleep_time in [1, 2, 3, 4, 5] for elem in list(gen)) pool_map_unordered = ActorPool([DummyActor.remote() for i in range(2)]) pool_map_unordered.submit(func, 6) start_time = time.time() gen = pool_map_unordered.map_unordered(func, map_values) assert all(elem[0] in [1, 2, 3, 4, 5] for elem in list(gen)) assert all( abs(elem[1] - start_time) < sleep_time in [1, 2, 3, 4, 5] for elem in list(gen))
def test_push(init): @ray.remote class MyActor: def __init__(self): pass def f(self, x): return x + 1 def double(self, x): return 2 * x a1, a2, a3 = MyActor.remote(), MyActor.remote(), MyActor.remote() pool = ActorPool([a1]) pool.submit(lambda a, v: a.double.remote(v), 1) assert pool.has_free() is False # actor is busy with pytest.raises(ValueError): pool.push(a1) pool.push(a2) assert pool.has_free() # a2 is available pool.submit(lambda a, v: a.double.remote(v), 1) pool.submit(lambda a, v: a.double.remote(v), 1) assert pool.has_free() is False assert len(pool._pending_submits) == 1 pool.push(a3) assert pool.has_free() is False # a3 is used for pending submit assert len(pool._pending_submits) == 0
def test_multiple_returns(init): @ray.remote class Foo(object): @ray.method(num_returns=2) def bar(self): return 1, 2 pool = ActorPool([Foo.remote() for _ in range(2)]) for _ in range(4): pool.submit(lambda a, v: a.bar.remote(), None) while pool.has_next(): assert pool.get_next(timeout=None) == [1, 2]
def test_get_next(init): @ray.remote class MyActor: def __init__(self): pass def f(self, x): return x + 1 def double(self, x): return 2 * x actors = [MyActor.remote() for _ in range(4)] pool = ActorPool(actors) for i in range(5): pool.submit(lambda a, v: a.f.remote(v), i) assert pool.get_next() == i + 1
def test_get_next_timeout(init): @ray.remote class MyActor: def __init__(self): pass def f(self, x): while True: x = x + 1 time.sleep(1) return None def double(self, x): return 2 * x actors = [MyActor.remote() for _ in range(4)] pool = ActorPool(actors) pool.submit(lambda a, v: a.f.remote(v), 0) with pytest.raises(TimeoutError): pool.get_next_unordered(timeout=0.1)
def remote_worker_caller(executor: ActorPool, sleep_time: int, number_invokes: int): start = time.time() [ executor.submit(lambda a, v: a.computation.remote(v), sleep_time) for n in range(number_invokes) ] starttime = time.time() - start while (executor.has_next()): executor.get_next() return starttime, time.time() - start
def tqdm_map(actors, actor_tup_function, tups, res=None): assert res is not None, "provide way to save partial results" initial_len = len(res) actor_pool = ActorPool(actors) for tup in tups: actor_pool.submit(actor_tup_function, tup) pbar = tqdm(total=len(tups)) while True: nxt = actor_pool.get_next_unordered() ## copy to free up any references at the source res.append(copy.deepcopy(nxt)) pbar.update(1) if (len(res) - initial_len) == len(tups): print("done with new tups") break pbar.close() return res
def test_map_gh23107(init): # Reference - https://github.com/ray-project/ray/issues/23107 @ray.remote class DummyActor: async def identity(self, s): return s def func(a, v): return a.identity.remote(v) map_values = [1, 2, 3, 4, 5] pool_map = ActorPool([DummyActor.remote() for i in range(2)]) pool_map.submit(func, 6) gen = pool_map.map(func, map_values) assert list(gen) == [1, 2, 3, 4, 5] pool_map_unordered = ActorPool([DummyActor.remote() for i in range(2)]) pool_map_unordered.submit(func, 6) gen = pool_map_unordered.map(func, map_values) assert all(elem in [1, 2, 3, 4, 5] for elem in list(gen))
def test_pop_idle(init): @ray.remote class MyActor: def __init__(self): pass def f(self, x): return x + 1 def double(self, x): return 2 * x actors = [MyActor.remote()] pool = ActorPool(actors) pool.submit(lambda a, v: a.double.remote(v), 1) assert pool.pop_idle() is None assert pool.has_free() is False # actor is busy assert pool.get_next() == 2 assert pool.has_free() pool.pop_idle() # removes actor from pool assert pool.has_free() is False # no more actors in pool
def test_get_next_unordered(init): @ray.remote class MyActor: def __init__(self): pass def f(self, x): return x + 1 def double(self, x): return 2 * x actors = [MyActor.remote() for _ in range(4)] pool = ActorPool(actors) total = [] for i in range(5): pool.submit(lambda a, v: a.f.remote(v), i) while pool.has_next(): total += [pool.get_next_unordered()] assert all(elem in [1, 2, 3, 4, 5] for elem in total)
@ray.remote class PoolActor: def __init__(self): self.id = str(uuid4()) def computation(self, num): print(f'Actor with id {self.id} waiting for {num} sec') for x in range(num): sleep(1) print(f'Actor with id {self.id} slept for {x} sec') return num # Create actors and add them to the pool a1, a2, a3 = PoolActor.remote(), PoolActor.remote(), PoolActor.remote() pool = ActorPool([a1, a2, a3]) print(list(pool.map(lambda a, v: a.computation.remote(v), [3, 4, 5, 4]))) pool.submit(lambda a, v: a.computation.remote(v), 3) pool.submit(lambda a, v: a.computation.remote(v), 4) pool.submit(lambda a, v: a.computation.remote(v), 5) pool.submit(lambda a, v: a.computation.remote(v), 4) print(pool.get_next()) print(pool.get_next()) print(pool.get_next()) print(pool.get_next())
def _main(cnf_dir, n_datapoints_per_file, dest, n_workers, dumpfreq=100e3, num_subproblems=8, random_units=5, timeout=300, rootdir=None, n_datapoints=25000): logger = SimpleLogger( os.path.join(dest, "logs", "main_loop", f"{str(uuid.uuid4())}.txt")) try: ray.init(address='auto', redis_password='******') except: ray.init() tms_dict = { cnf_path: TaskManager(cnf_path, rootdir) for cnf_path in recursively_get_files( cnf_dir, forbidden=["bz2", "xz"], exts=["cnf", "gz"]) } logger.write("STARTING DATA GENERATION LOOP WITH", len(tms_dict.keys()), "CNFs") time.sleep(5) writer = lbdpWriter.remote(n_datapoints_per_file, dest) workers = [ Worker.remote(writer, dumpfreq=dumpfreq, num_subproblems=num_subproblems, random_units=random_units, timeout=timeout) for _ in range(n_workers) ] pool = ActorPool(workers) for tm in tms_dict.values(): for task in tm: if task is not None: pool.submit((lambda a, v: a.work.remote(v)), task) logger.write( f"SUBMITTED TASK: TYPE {task.task_type}; CNF_PATH {task.cnf_path}" ) try: LOOP_COUNT = 0 while any([(x.waiting > 0) for x in tms_dict.values()]): LOOP_COUNT += 1 if LOOP_COUNT % 100 == 0: if ray.get(writer.get_write_count.remote()) > n_datapoints: print( f"NUMBER OF WRITES EXCEEDS N_DATAPOINTS={n_datapoints}, STOPPING" ) break cnf_path, tasks, original_task = pool.get_next_unordered() logger.write(f"GOT TASK RESULT (TYPE {original_task.task_type})") if original_task.task_type == 0 and len(tasks) == 0: logger.write("WARNING: Task", original_task.cnf_path, "returned no subtasks.") tm = tms_dict[cnf_path] tm.set_tasks(tasks) if (tm.waiting == 0) and len(tm.tasks) == 0: print(f"ROOT TASK {tm.cnf_path} FINISHED") tms_dict.pop(cnf_path) print("POPPED FROM TMS_DICT") try: shutil.rmtree(tms.tmpdir.name) os.makedirs(tms.tmpdir.name) time.sleep(1) except: pass else: for task in tm: if task.task_type == 0: logger.write(f"SUBMITTING SUBPROBLEM (TIMEOUT) TASK") elif task.task_type == 1: logger.write(f"SUBMITTING DERIVED FORMULA") elif task.task_type == 2: logger.write(f"SUBMITTING DERIVED SUBFORMULA") pool.submit((lambda a, v: a.work.remote(v)), task) logger.write(f"SUBMITTED {len(tasks)} NEW TASKS") finally: del writer