def test_execution_order(): L = [] for i in range(5): s = Stream() b = s.pluck(1) a = s.pluck(0) l = a.combine_latest(b, emit_on=a).sink_to_list() z = [(1, "red"), (2, "blue"), (3, "green")] for zz in z: s.emit(zz) L.append((l, )) for ll in L: assert ll == L[0] L2 = [] for i in range(5): s = Stream() a = s.pluck(0) b = s.pluck(1) l = a.combine_latest(b, emit_on=a).sink_to_list() z = [(1, "red"), (2, "blue"), (3, "green")] for zz in z: s.emit(zz) L2.append((l, )) for ll, ll2 in zip(L, L2): assert ll2 == L2[0] assert ll != ll2
def test_unique_list(): source = Stream() L = source.unique(history=1).sink_to_list() source.emit(["a"]) source.emit(["a"]) source.emit(["b"]) assert L == [["a"], ["b"]]
def test_unique_dict(): source = Stream() L = source.unique(history=1).sink_to_list() source.emit({"a": 1}) source.emit({"a": 1}) source.emit({"b": 1}) assert L == [{"a": 1}, {"b": 1}]
def test_filter_args_kwargs(): def f(x, y, z=False): print(y) print(z) return y and z source = Stream() L = source.filter(f, True, z=True).sink_to_list() source.emit(1) assert L[0] is 1
def test_starmap(): def add(x=0, y=0): return x + y source = Stream() L = source.starmap(add).sink_to_list() source.emit((1, 10)) assert L[0] == 11
def test_zip_latest_first(): a = Stream() b = Stream() c = a.zip_latest(b).starmap(op.sub) d = a.zip_latest(b, first=True).starmap(op.add) L = c.union(d).sink_to_list() a.emit(1) b.emit(1) assert L == [2, 0]
def test_sync_2(loop): with cluster() as (s, [a, b]): with Client(s["address"], loop=loop): # flake8: noqa source = Stream() L = source.scatter().map(inc).gather().sink_to_list() for i in range(10): source.emit(i) assert len(L) == i + 1 assert L == list(map(inc, range(10)))
def test_combine_latest_first(): a = Stream() b = Stream() c = a.zip(b) z = c.starmap(op.add) zz = z.combine_latest(b, emit_on=0, first=b) L = zz.sink_to_list() a.emit(1) b.emit(1) assert len(L) == 1
def test_star_sink(): L = [] def add(x, y): L.append(x + y) source = Stream() source.starsink(add) source.emit((1, 10)) assert L[0] == 11
def test_zip(c, s, a, b): a = Stream(asynchronous=True) b = Stream(asynchronous=True) c = scatter(a).zip(scatter(b)) L = c.gather().sink_to_list() yield a.emit(1) yield b.emit("a") yield a.emit(2) yield b.emit("b") assert L == [(1, "a"), (2, "b")]
def test_pluck(backend): source = Stream(asynchronous=True) L = source.scatter(backend=backend).pluck(0).gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == list(range(5))
def test_double_scatter(backend): source1 = Stream(asynchronous=True) source2 = Stream(asynchronous=True) sm = (source1.scatter(backend=backend).zip( source2.scatter(backend=backend)).starmap(add)) futures_L = sm.sink_to_list() r = sm.buffer(10).gather() L = r.sink_to_list() for i in range(5): yield source1.emit(i) yield source2.emit(i) while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [i + i for i in range(5)] assert all(isinstance(f, Future) for f in futures_L)
def test_starmap(c, s, a, b): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) L = source.scatter().starmap(add, z=10).gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == [10, 12, 14, 16, 18]
def test_scan(c, s, a, b): source = Stream(asynchronous=True) futures = scatter(source).map(inc).scan(add) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 3, 6, 10, 15] assert all(isinstance(f, Future) for f in futures_L)
def test_map(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend).map(inc) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 2, 3, 4, 5] assert all(isinstance(f, Future) for f in futures_L)
def test_combined_latest(backend): def delay(x): time.sleep(.5) return x source = Stream(asynchronous=True) source2 = Stream(asynchronous=True) futures = (source.scatter(backend=backend).map(delay).combine_latest( source2.scatter(backend=backend), emit_on=1)) futures_L = futures.sink_to_list() L = futures.buffer(10).gather().sink_to_list() for i in range(5): yield source.emit(i) yield source.emit(i) yield source2.emit(i) while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [(i, i) for i in range(5)]
def test_filter_map(c, s, a, b): source = Stream(asynchronous=True) futures = scatter(source).filter(lambda x: x % 2 == 0).map(inc) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [1, 3, 5] assert all(isinstance(f, Future) for f in futures_L)
def test_filter(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend).filter(lambda x: x % 2 == 0) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [0, 2, 4] assert all(isinstance(f, Future) for f in futures_L)
def test_filter_zip(c, s, a, b): source = Stream(asynchronous=True) s = scatter(source) futures = s.filter(lambda x: x % 2 == 0).zip(s) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert L == [(a, a) for a in [0, 2, 4]] assert all(isinstance(f[0], Future) for f in futures_L)
def test_filter_starmap(backend): source = Stream(asynchronous=True) futures1 = scatter(source, backend=backend).filter(lambda x: x[1] % 2 == 0) futures = futures1.starmap(add) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert L == [0, 4, 8] assert all(isinstance(f, Future) for f in futures_L)
def test_scan_state(c, s, a, b): source = Stream(asynchronous=True) def f(acc, i): acc = acc + i return acc, acc L = scatter(source).scan(f, returns_state=True).gather().sink_to_list() for i in range(3): yield source.emit(i) assert L == [0, 1, 3]
def test_buffer(backend): source = Stream(asynchronous=True) L = (source.scatter(backend=backend).map( slowinc, delay=0.5).buffer(5).gather().sink_to_list()) start = time.time() for i in range(5): yield source.emit(i) end = time.time() assert end - start < 0.5 for i in range(5, 10): yield source.emit(i) end2 = time.time() assert end2 - start > (0.5 / 3) while len(L) < 10: yield gen.sleep(0.01) assert time.time() - start < 5 assert L == list(map(inc, range(10)))
def test_buffer_sync(loop): with cluster() as (s, [a, b]): with Client(s["address"], loop=loop) as c: # flake8: noqa source = Stream() buff = source.scatter().map(slowinc, delay=0.5).buffer(5) L = buff.gather().sink_to_list() start = time.time() for i in range(5): source.emit(i) end = time.time() assert end - start < 0.5 for i in range(5, 10): source.emit(i) end2 = time.time() while len(L) < 10: time.sleep(0.01) assert time.time() - start < 5 assert L == list(map(inc, range(10)))
def test_double_scatter(c, s, a, b): source1 = Stream(asynchronous=True) source1.sink(print) source2 = Stream(asynchronous=True) source2.sink(print) sm = source1.scatter().zip(source2.scatter()).starmap(add) futures_L = sm.sink_to_list() r = sm.buffer(10).gather() L = r.sink_to_list() print("hi") for i in range(5): print("hi") yield source1.emit(i) yield source2.emit(i) while len(L) < len(futures_L): print(len(L), print(len(futures_L))) yield gen.sleep(.01) assert L == [i + i for i in range(5)] assert all(isinstance(f, Future) for f in futures_L)
def test_starmap(backend): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) futures = scatter(source, backend=backend).starmap(add, z=10) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit((i, i)) assert len(L) == len(futures_L) assert L == [10, 12, 14, 16, 18]
def test_buffer2(backend): source = Stream(asynchronous=True) futures = scatter(source, backend=backend) futures_L = futures.sink_to_list() L = futures.buffer(10).gather().sink_to_list() for i in range(5): yield source.emit(i) assert len(futures_L) == 5 while len(L) < len(futures_L): yield gen.sleep(.01) assert L == [0, 1, 2, 3, 4] assert all(isinstance(f, Future) for f in futures_L)
def test_starmap_args(c, s, a, b): def add(x, y, z=0): return x + y + z source = Stream(asynchronous=True) futures = scatter(source).starmap(add, 10) futures_L = futures.sink_to_list() L = futures.gather().sink_to_list() for i in range(5): yield source.emit(i) assert len(L) == len(futures_L) assert L == [i + 10 for i in range(5)]
def print_sleep(x): plt.pause(.1) print(x) b = source.map(sleep_inc) b.sink(print_sleep) c = b.map(sleep_inc) c.sink(print_sleep) gv = run_vis( source, source_node=True, edge_style={"color": "k"}, node_label_style={"font_size": 15}, edge_label_style=lambda x: {"label": x["label"], "font_size": 15}, node_style=node_style, force_draw=True, ) plt.pause(.1) for i in range(10): try: source.emit(i) # plt.pause(10) except RuntimeError: pass plt.show()