def test_zip_latest_ref_counts():
a = Stream()
b = Stream()
_ = a.zip_latest(b)
ref1 = RefCounter()
a.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1 # Retained until stream b has a value
# The lossless stream is never retained if all upstreams have a value
ref2 = RefCounter()
b.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
# Kick out the stream b value and verify it has zero references
ref3 = RefCounter()
b.emit(3, metadata=[{'ref': ref3}])
assert ref2.count == 0
assert ref3.count == 1
# Verify the lossless value is not retained, but the lossy value is
ref4 = RefCounter()
a.emit(3, metadata=[{'ref': ref4}])
assert ref3.count == 1
assert ref4.count == 0
def test_latest_ref_counts():
source = Stream()
_ = source.latest()
ref1 = RefCounter()
source.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
ref2 = RefCounter()
source.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
def test_timed_window_ref_counts():
source = Stream()
_ = source.timed_window(0.01)
ref1 = RefCounter()
source.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
yield gen.sleep(0.05)
ref2 = RefCounter()
source.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
def test_sliding_window_ref_counts():
source = Stream()
_ = source.sliding_window(2)
r_prev = RefCounter()
source.emit(-2)
source.emit(-1, metadata=[{'ref': r_prev}])
for i in range(10):
r = RefCounter()
assert r_prev.count == 1
source.emit(i, metadata=[{'ref': r}])
assert r_prev.count == 0
assert r.count == 1
r_prev = r
def test_timed_window_ref_counts():
source = Stream(asynchronous=True)
_ = source.timed_window(0.01)
ref1 = RefCounter()
assert str(ref1) == "<RefCounter count=0>"
source.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
yield gen.sleep(0.05)
ref2 = RefCounter()
source.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
def test_partition_ref_counts():
source = Stream()
_ = source.partition(2)
for i in range(10):
r = RefCounter()
source.emit(i, metadata=[{'ref': r}])
if i % 2 == 0:
assert r.count == 1
else:
assert r.count == 0
def test_combine_latest_ref_counts():
a = Stream()
b = Stream()
_ = a.combine_latest(b)
ref1 = RefCounter()
a.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
# The new value kicks out the old value
ref2 = RefCounter()
a.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
# The value on stream a is still retained and the value on stream b is new
ref3 = RefCounter()
b.emit(3, metadata=[{'ref': ref3}])
assert ref2.count == 1
assert ref3.count == 1
def test_zip_ref_counts():
a = Stream()
b = Stream()
_ = a.zip(b)
# The first value in a becomes buffered
ref1 = RefCounter()
a.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
# The second value in a also becomes buffered
ref2 = RefCounter()
a.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 1
assert ref2.count == 1
# All emitted values are removed from the buffer
ref3 = RefCounter()
b.emit(3, metadata=[{'ref': ref3}])
assert ref1.count == 0
assert ref2.count == 1 # still in the buffer
assert ref3.count == 0
def test_buffer_ref_counts():
source = Stream(asynchronous=True)
_ = source.buffer(5)
refs = []
for i in range(5):
r = RefCounter()
refs.append(r)
source.emit(i, metadata=[{'ref': r}])
assert all(r.count == 1 for r in refs)
yield gen.sleep(0.05)
assert all(r.count == 0 for r in refs)
def test_collect_ref_counts():
source = Stream()
collector = source.collect()
refs = []
for i in range(10):
r = RefCounter()
refs.append(r)
source.emit(i, metadata=[{'ref': r}])
assert all(r.count == 1 for r in refs)
collector.flush()
assert all(r.count == 0 for r in refs)
async def test_partition_then_scatter_async(c, s, a, b):
# Ensure partition w/ timeout before scatter works correctly for
# asynchronous
start = time.monotonic()
source = Stream(asynchronous=True)
L = source.partition(
2, timeout=.1).scatter().map(lambda x: [xx + 1 for xx in x]).buffer(
2).gather().flatten().sink_to_list()
rc = RefCounter(loop=source.loop)
for i in range(3):
await source.emit(i, metadata=[{'ref': rc}])
while rc.count != 0 and time.monotonic() - start < 1.:
await gen.sleep(1e-2)
assert L == [1, 2, 3]
def test_partition_then_scatter_sync(loop):
# Ensure partition w/ timeout before scatter works correctly for synchronous
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as client: # noqa: F841
start = time.monotonic()
source = Stream()
L = source.partition(
2,
timeout=.1).scatter().map(lambda x: [xx + 1 for xx in x]
).gather().flatten().sink_to_list()
assert source.loop is client.loop
rc = RefCounter()
for i in range(3):
source.emit(i, metadata=[{'ref': rc}])
while rc.count != 0 and time.monotonic() - start < 2.:
time.sleep(1e-2)
assert L == [1, 2, 3]
