def test_execution_order():
L = []
for i in range(5):
s = Stream()
b = s.pluck(1)
a = s.pluck(0)
l = a.combine_latest(b, emit_on=a).sink_to_list()
z = [(1, "red"), (2, "blue"), (3, "green")]
for zz in z:
s.emit(zz)
L.append((l, ))
for ll in L:
assert ll == L[0]
L2 = []
for i in range(5):
s = Stream()
a = s.pluck(0)
b = s.pluck(1)
l = a.combine_latest(b, emit_on=a).sink_to_list()
z = [(1, "red"), (2, "blue"), (3, "green")]
for zz in z:
s.emit(zz)
L2.append((l, ))
for ll, ll2 in zip(L, L2):
assert ll2 == L2[0]
assert ll != ll2
def test_unique_list():
source = Stream()
L = source.unique(history=1).sink_to_list()
source.emit(["a"])
source.emit(["a"])
source.emit(["b"])
assert L == [["a"], ["b"]]
def test_unique_dict():
source = Stream()
L = source.unique(history=1).sink_to_list()
source.emit({"a": 1})
source.emit({"a": 1})
source.emit({"b": 1})
assert L == [{"a": 1}, {"b": 1}]
def test_filter_args_kwargs():
def f(x, y, z=False):
print(y)
print(z)
return y and z
source = Stream()
L = source.filter(f, True, z=True).sink_to_list()
source.emit(1)
assert L[0] is 1
def test_starmap():
def add(x=0, y=0):
return x + y
source = Stream()
L = source.starmap(add).sink_to_list()
source.emit((1, 10))
assert L[0] == 11
def test_zip_latest_first():
a = Stream()
b = Stream()
c = a.zip_latest(b).starmap(op.sub)
d = a.zip_latest(b, first=True).starmap(op.add)
L = c.union(d).sink_to_list()
a.emit(1)
b.emit(1)
assert L == [2, 0]
def test_sync_2(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop): # flake8: noqa
source = Stream()
L = source.scatter().map(inc).gather().sink_to_list()
for i in range(10):
source.emit(i)
assert len(L) == i + 1
assert L == list(map(inc, range(10)))
def test_combine_latest_first():
a = Stream()
b = Stream()
c = a.zip(b)
z = c.starmap(op.add)
zz = z.combine_latest(b, emit_on=0, first=b)
L = zz.sink_to_list()
a.emit(1)
b.emit(1)
assert len(L) == 1
def test_star_sink():
L = []
def add(x, y):
L.append(x + y)
source = Stream()
source.starsink(add)
source.emit((1, 10))
assert L[0] == 11
def test_zip(c, s, a, b):
a = Stream(asynchronous=True)
b = Stream(asynchronous=True)
c = scatter(a).zip(scatter(b))
L = c.gather().sink_to_list()
yield a.emit(1)
yield b.emit("a")
yield a.emit(2)
yield b.emit("b")
assert L == [(1, "a"), (2, "b")]
def test_pluck(backend):
source = Stream(asynchronous=True)
L = source.scatter(backend=backend).pluck(0).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == list(range(5))
def test_double_scatter(backend):
source1 = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
sm = (source1.scatter(backend=backend).zip(
source2.scatter(backend=backend)).starmap(add))
futures_L = sm.sink_to_list()
r = sm.buffer(10).gather()
L = r.sink_to_list()
for i in range(5):
yield source1.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [i + i for i in range(5)]
assert all(isinstance(f, Future) for f in futures_L)
def test_starmap(c, s, a, b):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
L = source.scatter().starmap(add, z=10).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == [10, 12, 14, 16, 18]
def test_scan(c, s, a, b):
source = Stream(asynchronous=True)
futures = scatter(source).map(inc).scan(add)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 3, 6, 10, 15]
assert all(isinstance(f, Future) for f in futures_L)
def test_map(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).map(inc)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 2, 3, 4, 5]
assert all(isinstance(f, Future) for f in futures_L)
def test_combined_latest(backend):
def delay(x):
time.sleep(.5)
return x
source = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
futures = (source.scatter(backend=backend).map(delay).combine_latest(
source2.scatter(backend=backend), emit_on=1))
futures_L = futures.sink_to_list()
L = futures.buffer(10).gather().sink_to_list()
for i in range(5):
yield source.emit(i)
yield source.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [(i, i) for i in range(5)]
def test_filter_map(c, s, a, b):
source = Stream(asynchronous=True)
futures = scatter(source).filter(lambda x: x % 2 == 0).map(inc)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [1, 3, 5]
assert all(isinstance(f, Future) for f in futures_L)
def test_filter(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).filter(lambda x: x % 2 == 0)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [0, 2, 4]
assert all(isinstance(f, Future) for f in futures_L)
def test_filter_zip(c, s, a, b):
source = Stream(asynchronous=True)
s = scatter(source)
futures = s.filter(lambda x: x % 2 == 0).zip(s)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert L == [(a, a) for a in [0, 2, 4]]
assert all(isinstance(f[0], Future) for f in futures_L)
def test_filter_starmap(backend):
source = Stream(asynchronous=True)
futures1 = scatter(source, backend=backend).filter(lambda x: x[1] % 2 == 0)
futures = futures1.starmap(add)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == [0, 4, 8]
assert all(isinstance(f, Future) for f in futures_L)
def test_scan_state(c, s, a, b):
source = Stream(asynchronous=True)
def f(acc, i):
acc = acc + i
return acc, acc
L = scatter(source).scan(f, returns_state=True).gather().sink_to_list()
for i in range(3):
yield source.emit(i)
assert L == [0, 1, 3]
def test_buffer(backend):
source = Stream(asynchronous=True)
L = (source.scatter(backend=backend).map(
slowinc, delay=0.5).buffer(5).gather().sink_to_list())
start = time.time()
for i in range(5):
yield source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
yield source.emit(i)
end2 = time.time()
assert end2 - start > (0.5 / 3)
while len(L) < 10:
yield gen.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))
def test_buffer_sync(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop) as c: # flake8: noqa
source = Stream()
buff = source.scatter().map(slowinc, delay=0.5).buffer(5)
L = buff.gather().sink_to_list()
start = time.time()
for i in range(5):
source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
source.emit(i)
end2 = time.time()
while len(L) < 10:
time.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))
def test_double_scatter(c, s, a, b):
source1 = Stream(asynchronous=True)
source1.sink(print)
source2 = Stream(asynchronous=True)
source2.sink(print)
sm = source1.scatter().zip(source2.scatter()).starmap(add)
futures_L = sm.sink_to_list()
r = sm.buffer(10).gather()
L = r.sink_to_list()
print("hi")
for i in range(5):
print("hi")
yield source1.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
print(len(L), print(len(futures_L)))
yield gen.sleep(.01)
assert L == [i + i for i in range(5)]
assert all(isinstance(f, Future) for f in futures_L)
def test_starmap(backend):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend).starmap(add, z=10)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert len(L) == len(futures_L)
assert L == [10, 12, 14, 16, 18]
def test_buffer2(backend):
source = Stream(asynchronous=True)
futures = scatter(source, backend=backend)
futures_L = futures.sink_to_list()
L = futures.buffer(10).gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert len(futures_L) == 5
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [0, 1, 2, 3, 4]
assert all(isinstance(f, Future) for f in futures_L)
def test_starmap_args(c, s, a, b):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
futures = scatter(source).starmap(add, 10)
futures_L = futures.sink_to_list()
L = futures.gather().sink_to_list()
for i in range(5):
yield source.emit(i)
assert len(L) == len(futures_L)
assert L == [i + 10 for i in range(5)]
def print_sleep(x):
plt.pause(.1)
print(x)
b = source.map(sleep_inc)
b.sink(print_sleep)
c = b.map(sleep_inc)
c.sink(print_sleep)
gv = run_vis(
source,
source_node=True,
edge_style={"color": "k"},
node_label_style={"font_size": 15},
edge_label_style=lambda x: {"label": x["label"], "font_size": 15},
node_style=node_style,
force_draw=True,
)
plt.pause(.1)
for i in range(10):
try:
source.emit(i)
# plt.pause(10)
except RuntimeError:
pass
plt.show()