def test_pluck(backend):
source = Stream(asynchronous=True)
L = source.scatter(backend=backend).pluck(0).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == list(range(5))
def test_double_scatter(backend):
source1 = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
sm = (source1.scatter(backend=backend).zip(
source2.scatter(backend=backend)).starmap(add))
futures_L = sm.sink_to_list()
r = sm.buffer(10).gather()
L = r.sink_to_list()
for i in range(5):
yield source1.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [i + i for i in range(5)]
assert all(isinstance(f, Future) for f in futures_L)
def test_starmap(c, s, a, b):
def add(x, y, z=0):
return x + y + z
source = Stream(asynchronous=True)
L = source.scatter().starmap(add, z=10).gather().sink_to_list()
for i in range(5):
yield source.emit((i, i))
assert L == [10, 12, 14, 16, 18]
def test_sync_2(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop): # flake8: noqa
source = Stream()
L = source.scatter().map(inc).gather().sink_to_list()
for i in range(10):
source.emit(i)
assert len(L) == i + 1
assert L == list(map(inc, range(10)))
def test_combined_latest(backend):
def delay(x):
time.sleep(.5)
return x
source = Stream(asynchronous=True)
source2 = Stream(asynchronous=True)
futures = (source.scatter(backend=backend).map(delay).combine_latest(
source2.scatter(backend=backend), emit_on=1))
futures_L = futures.sink_to_list()
L = futures.buffer(10).gather().sink_to_list()
for i in range(5):
yield source.emit(i)
yield source.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
yield gen.sleep(.01)
assert L == [(i, i) for i in range(5)]
def test_double_scatter(c, s, a, b):
source1 = Stream(asynchronous=True)
source1.sink(print)
source2 = Stream(asynchronous=True)
source2.sink(print)
sm = source1.scatter().zip(source2.scatter()).starmap(add)
futures_L = sm.sink_to_list()
r = sm.buffer(10).gather()
L = r.sink_to_list()
print("hi")
for i in range(5):
print("hi")
yield source1.emit(i)
yield source2.emit(i)
while len(L) < len(futures_L):
print(len(L), print(len(futures_L)))
yield gen.sleep(.01)
assert L == [i + i for i in range(5)]
assert all(isinstance(f, Future) for f in futures_L)
def test_sync(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop) as client: # flake8: noqa
source = Stream()
L = source.scatter().map(inc).gather().sink_to_list()
@gen.coroutine
def f():
for i in range(10):
yield source.emit(i, asynchronous=True)
sync(loop, f)
assert L == list(map(inc, range(10)))
def test_buffer_sync(loop):
with cluster() as (s, [a, b]):
with Client(s["address"], loop=loop) as c: # flake8: noqa
source = Stream()
buff = source.scatter().map(slowinc, delay=0.5).buffer(5)
L = buff.gather().sink_to_list()
start = time.time()
for i in range(5):
source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
source.emit(i)
end2 = time.time()
while len(L) < 10:
time.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))
def test_buffer(backend):
source = Stream(asynchronous=True)
L = (source.scatter(backend=backend).map(
slowinc, delay=0.5).buffer(5).gather().sink_to_list())
start = time.time()
for i in range(5):
yield source.emit(i)
end = time.time()
assert end - start < 0.5
for i in range(5, 10):
yield source.emit(i)
end2 = time.time()
assert end2 - start > (0.5 / 3)
while len(L) < 10:
yield gen.sleep(0.01)
assert time.time() - start < 5
assert L == list(map(inc, range(10)))