def f(block, ps=None): start = time() params = ps.get_data(separate_thread=False).result() stop = time() update = (block - params).mean(axis=0) ps.update(update, separate_thread=False) print(format_time(stop - start)) return np.array([[stop - start]])
def f(block, ps=None): start = time() params = ps.get_data(separate_thread=False).result() stop = time() update = (block - params).mean(axis=0) ps.update(update, separate_thread=False) print(format_time(stop - start)) return np.array([[stop - start]])
def update(self): with log_errors(): outgoing = self.worker.outgoing_transfer_log n = self.worker.outgoing_count - self.last_outgoing outgoing = [outgoing[-i].copy() for i in range(1, n + 1)] self.last_outgoing = self.worker.outgoing_count incoming = self.worker.incoming_transfer_log n = self.worker.incoming_count - self.last_incoming incoming = [incoming[-i].copy() for i in range(1, n + 1)] self.last_incoming = self.worker.incoming_count for [msgs, source] in [ [incoming, self.incoming], [outgoing, self.outgoing], ]: for msg in msgs: if "compressed" in msg: del msg["compressed"] del msg["keys"] bandwidth = msg["total"] / (msg["duration"] or 0.5) bw = max(min(bandwidth / 500e6, 1), 0.3) msg["alpha"] = bw try: msg["y"] = self.who[msg["who"]] except KeyError: self.who[msg["who"]] = len(self.who) msg["y"] = self.who[msg["who"]] msg["hover"] = "%s / %s = %s/s" % ( format_bytes(msg["total"]), format_time(msg["duration"]), format_bytes(msg["total"] / msg["duration"]), ) for k in ["middle", "duration", "start", "stop"]: msg[k] = msg[k] * 1000 if msgs: msgs = transpose(msgs) if (len(source.data["stop"]) and min(msgs["start"]) > source.data["stop"][-1] + 10000): source.data.update(msgs) else: source.stream(msgs, rollover=10000)
def bench_param_server(c, s, *workers): import dask.array as da import numpy as np x = da.random.random((500000, 1000), chunks=(1000, 1000)) x = x.persist() yield wait(x) class ParameterServer: data = None def __init__(self, n): self.data = np.random.random(n) def update(self, x): self.data += x self.data /= 2 def get_data(self): return self.data def f(block, ps=None): start = time() params = ps.get_data(separate_thread=False).result() stop = time() update = (block - params).mean(axis=0) ps.update(update, separate_thread=False) print(format_time(stop - start)) return np.array([[stop - start]]) from distributed.utils import format_time start = time() ps = yield c.submit(ParameterServer, x.shape[1], actor=True) y = x.map_blocks(f, ps=ps, dtype=x.dtype) # result = yield c.compute(y.mean()) yield wait(y.persist()) end = time() print(format_time(end - start))
def bench_param_server(c, s, *workers): import dask.array as da import numpy as np x = da.random.random((500000, 1000), chunks=(1000, 1000)) x = x.persist() yield wait(x) class ParameterServer: data = None def __init__(self, n): self.data = np.random.random(n) def update(self, x): self.data += x self.data /= 2 def get_data(self): return self.data def f(block, ps=None): start = time() params = ps.get_data(separate_thread=False).result() stop = time() update = (block - params).mean(axis=0) ps.update(update, separate_thread=False) print(format_time(stop - start)) return np.array([[stop - start]]) from distributed.utils import format_time start = time() ps = yield c.submit(ParameterServer, x.shape[1], actor=True) y = x.map_blocks(f, ps=ps, dtype=x.dtype) # result = yield c.compute(y.mean()) yield wait(y.persist()) end = time() print(format_time(end - start))