def test_delete_shm(): initial_dev_shm = os.listdir("/dev/shm") cfg = tsk.Cfg(initializer=setup_worker) r = tsk.zmq_run(cfg=cfg, f=f) assert r == 99 final_dev_shm = os.listdir("/dev/shm") new_files = set(final_dev_shm) - set(initial_dev_shm) assert len(new_files) == 0
def run_tsk_parallel(): async def f(): gang = await tsk.ctx().wait_for_workers(2) fnc_dref = tsk.put(long_fnc) async with tsk.Profiler(gang): start = time.time() await wait_all([ tsk.task(fnc_dref, to=gang[i % len(gang)]) for i in range(n_jobs) ]) print("inside: ", time.time() - start) def setup_worker(name): tsk.cfg.stdout_logging(name, "taskloaf.profile") cfg = tsk.Cfg(initializer=setup_worker) return tsk.zmq_run(cfg=cfg, f=f)
def main(): cfg = tsk.Cfg(n_workers=2, log_level=logging.WARN, initializer=setup_worker) async def f(): nrows = int(1e7) nnz = nrows * 10 n_repeats = 1 mat = random_test_matrix(nrows, nnz) vec = np.random.rand(nrows) - 0.5 t = tsk.Timer() for i in range(n_repeats): correct = mat.dot(vec) t.report("simple dot") gang = await tsk.ctx().wait_for_workers(cfg.n_workers) t.report("wait for workers") t.report("launch profiler") tsk_vec = TskArray(vals=vec) t.report("shmem v") tsk_mat = distribute(mat, gang) t.report("distribute mat") result = await tsk_mat.dot(tsk_vec) t.report("first dot") async with tsk.Profiler(gang): t.restart() for i in range(n_repeats): result = await tsk_mat.dot(tsk_vec) t.report("parallel dot") print(np.sum(correct)) print(np.sum(result)) assert np.sum(result) == np.sum(correct) tsk.zmq_run(cfg=cfg, f=f)
import sys import taskloaf as tsk if __name__ == "__main__": cfg = tsk.Cfg() cfg.n_workers = int(sys.argv[1]) cfg.base_port = int(sys.argv[2]) if len(sys.argv) > 3: connect_port = int(sys.argv[3]) cfg.connect_to = (cfg.hostname, connect_port) tsk.zmq_run(cfg=cfg)
import taskloaf as tsk import logging async def submit(): gang = await tsk.ctx().wait_for_workers(2) X = 3.1 n = 10 pr = tsk.task(lambda: tsk.task(lambda: X)) for i in range(n): pr = pr.then(lambda x: x + 1) async def asum(x): return sum(x) pr = tsk.when_all([pr, tsk.task(lambda: X, to=gang[1])]).then(asum) print("answer is", await pr) if __name__ == "__main__": cfg = tsk.Cfg(log_level=logging.WARN) tsk.zmq_run(cfg=cfg, f=submit)
import numpy as np import taskloaf as tsk async def submit(): gang = await tsk.ctx().wait_for_workers(2) n = int(4e8) ref = tsk.alloc(n * 8) A = np.frombuffer(await ref.get(), dtype=np.float64) A[:] = np.random.rand(n) rhs = np.sum(A) for i in range(2): async with tsk.Profiler(gang): # ref = tsk.put(w, A.data.cast('B')) async def remote(): A = np.frombuffer(await ref.get(), dtype=np.float64) return np.sum(A) lhs = await tsk.task(remote, to=gang[1]) assert lhs == rhs def setup_worker(name, cfg): tsk.cfg.stdout_logging(name, "taskloaf.profile") if __name__ == "__main__": cfg = tsk.Cfg(initializer=setup_worker) tsk.zmq_run(cfg=cfg, f=submit)
import taskloaf as tsk N = 4 async def f(): gang = await tsk.ctx().wait_for_workers(N) await tsk.task(lambda: print("hI"), to=gang[N - 2]) return 123 if __name__ == "__main__": for i in range(2): r = tsk.zmq_run(cfg=tsk.Cfg(n_workers=N), f=f) print(r) assert r == 123