def test_add_remove_worker(s): events = [] class MyPlugin(SchedulerPlugin): def add_worker(self, worker, scheduler): assert scheduler is s events.append(("add_worker", worker)) def remove_worker(self, worker, scheduler): assert scheduler is s events.append(("remove_worker", worker)) plugin = MyPlugin() s.add_plugin(plugin) assert events == [] a = Worker(s.address) b = Worker(s.address) yield a yield b yield a.close() yield b.close() assert events == [ ("add_worker", a.address), ("add_worker", b.address), ("remove_worker", a.address), ("remove_worker", b.address), ] events[:] = [] s.remove_plugin(plugin) a = yield Worker(s.address) yield a.close() assert events == []
def test_no_workers_to_memory(c, s): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.address, nthreads=1) w.put_key_in_memory(y.key, 3) yield w start = time() while not s.workers: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: "released", y.key: "memory", z.key: "processing", } yield w.close()
def test_no_worker_to_memory_restrictions(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z], workers={(x, y, z): "alice"}) while not s.tasks: yield gen.sleep(0.01) w = Worker(s.address, nthreads=1, name="alice") w.put_key_in_memory(y.key, 3) yield w while len(s.workers) < 3: yield gen.sleep(0.01) yield gen.sleep(0.3) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: "released", y.key: "memory", z.key: "processing", } yield w.close()
def test_worker_arrives_with_processing_data(c, s, a, b): x = delayed(slowinc)(1, delay=0.4) y = delayed(slowinc)(x, delay=0.4) z = delayed(slowinc)(y, delay=0.4) yy, zz = c.persist([y, z]) while not any(w.processing for w in s.workers.values()): yield gen.sleep(0.01) w = Worker(s.address, nthreads=1) w.put_key_in_memory(y.key, 3) yield w start = time() while len(s.workers) < 3: yield gen.sleep(0.01) assert s.get_task_status(keys={x.key, y.key, z.key}) == { x.key: "released", y.key: "memory", z.key: "processing", } yield w.close()
def test_add_worker(s, a, b): w = Worker(s.ip, s.port, ncores=3) w.data["x-5"] = 6 w.data["y"] = 1 yield w dsk = {("x-%d" % i): (inc, i) for i in range(10)} s.update_graph( tasks=valmap(dumps_task, dsk), keys=list(dsk), client="client", dependencies={k: set() for k in dsk}, ) s.add_worker(address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services) s.validate_state() assert w.ip in s.host_info assert s.host_info[w.ip]["addresses"] == {a.address, b.address, w.address} yield w.close()
def f(): s = Scheduler(loop=loop) done = s.start(0) a = Worker(s.ip, s.port, loop=loop, ncores=1) b = Worker(s.ip, s.port, loop=loop, ncores=1) yield [a._start(0), b._start(0)] progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False, interval=0.01) yield progress.listen() assert progress.status == "finished" check_bar_completed(capsys) yield [a.close(), b.close()] s.close() yield done
def test_sync_interactively(): c = Center('127.0.0.1', 8017, start=True, block=False) a = Worker('127.0.0.1', 8018, c.ip, c.port, ncores=1, start=True, block=False) b = Worker('127.0.0.1', 8019, c.ip, c.port, ncores=1, start=True, block=False) try: while len(c.ncores) < 2: sleep(0.01) values = [1, 2, 3, 4, 5, 6, 7, 8] data = sync(scatter_to_center(c.ip, c.port, values)) assert merge(a.data, b.data) == {d.key: v for d, v in zip(data, values)} results = sync(collect_from_center(c.ip, c.port, data)) assert results == values finally: a.close() b.close() c.close()
def test_scatter_no_workers(c, s): with pytest.raises(gen.TimeoutError): yield s.scatter(data={"x": 1}, client="alice", timeout=0.1) start = time() with pytest.raises(gen.TimeoutError): yield c.scatter(123, timeout=0.1) assert time() < start + 1.5 w = Worker(s.address, nthreads=3) yield [c.scatter(data={"y": 2}, timeout=5), w] assert w.data["y"] == 2 yield w.close()