def test_zip_latest_ref_counts(): a = Stream() b = Stream() _ = a.zip_latest(b) ref1 = RefCounter() a.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 # Retained until stream b has a value # The lossless stream is never retained if all upstreams have a value ref2 = RefCounter() b.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 0 assert ref2.count == 1 # Kick out the stream b value and verify it has zero references ref3 = RefCounter() b.emit(3, metadata=[{'ref': ref3}]) assert ref2.count == 0 assert ref3.count == 1 # Verify the lossless value is not retained, but the lossy value is ref4 = RefCounter() a.emit(3, metadata=[{'ref': ref4}]) assert ref3.count == 1 assert ref4.count == 0
def test_latest_ref_counts(): source = Stream() _ = source.latest() ref1 = RefCounter() source.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 ref2 = RefCounter() source.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 0 assert ref2.count == 1
def test_timed_window_ref_counts(): source = Stream() _ = source.timed_window(0.01) ref1 = RefCounter() source.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 yield gen.sleep(0.05) ref2 = RefCounter() source.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 0 assert ref2.count == 1
def test_sliding_window_ref_counts(): source = Stream() _ = source.sliding_window(2) r_prev = RefCounter() source.emit(-2) source.emit(-1, metadata=[{'ref': r_prev}]) for i in range(10): r = RefCounter() assert r_prev.count == 1 source.emit(i, metadata=[{'ref': r}]) assert r_prev.count == 0 assert r.count == 1 r_prev = r
def test_timed_window_ref_counts(): source = Stream(asynchronous=True) _ = source.timed_window(0.01) ref1 = RefCounter() assert str(ref1) == "<RefCounter count=0>" source.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 yield gen.sleep(0.05) ref2 = RefCounter() source.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 0 assert ref2.count == 1
def test_partition_ref_counts(): source = Stream() _ = source.partition(2) for i in range(10): r = RefCounter() source.emit(i, metadata=[{'ref': r}]) if i % 2 == 0: assert r.count == 1 else: assert r.count == 0
def test_combine_latest_ref_counts(): a = Stream() b = Stream() _ = a.combine_latest(b) ref1 = RefCounter() a.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 # The new value kicks out the old value ref2 = RefCounter() a.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 0 assert ref2.count == 1 # The value on stream a is still retained and the value on stream b is new ref3 = RefCounter() b.emit(3, metadata=[{'ref': ref3}]) assert ref2.count == 1 assert ref3.count == 1
def test_zip_ref_counts(): a = Stream() b = Stream() _ = a.zip(b) # The first value in a becomes buffered ref1 = RefCounter() a.emit(1, metadata=[{'ref': ref1}]) assert ref1.count == 1 # The second value in a also becomes buffered ref2 = RefCounter() a.emit(2, metadata=[{'ref': ref2}]) assert ref1.count == 1 assert ref2.count == 1 # All emitted values are removed from the buffer ref3 = RefCounter() b.emit(3, metadata=[{'ref': ref3}]) assert ref1.count == 0 assert ref2.count == 1 # still in the buffer assert ref3.count == 0
def test_buffer_ref_counts(): source = Stream(asynchronous=True) _ = source.buffer(5) refs = [] for i in range(5): r = RefCounter() refs.append(r) source.emit(i, metadata=[{'ref': r}]) assert all(r.count == 1 for r in refs) yield gen.sleep(0.05) assert all(r.count == 0 for r in refs)
def test_collect_ref_counts(): source = Stream() collector = source.collect() refs = [] for i in range(10): r = RefCounter() refs.append(r) source.emit(i, metadata=[{'ref': r}]) assert all(r.count == 1 for r in refs) collector.flush() assert all(r.count == 0 for r in refs)
async def test_partition_then_scatter_async(c, s, a, b): # Ensure partition w/ timeout before scatter works correctly for # asynchronous start = time.monotonic() source = Stream(asynchronous=True) L = source.partition( 2, timeout=.1).scatter().map(lambda x: [xx + 1 for xx in x]).buffer( 2).gather().flatten().sink_to_list() rc = RefCounter(loop=source.loop) for i in range(3): await source.emit(i, metadata=[{'ref': rc}]) while rc.count != 0 and time.monotonic() - start < 1.: await gen.sleep(1e-2) assert L == [1, 2, 3]
def test_partition_then_scatter_sync(loop): # Ensure partition w/ timeout before scatter works correctly for synchronous with cluster() as (s, [a, b]): with Client(s['address'], loop=loop) as client: # noqa: F841 start = time.monotonic() source = Stream() L = source.partition( 2, timeout=.1).scatter().map(lambda x: [xx + 1 for xx in x] ).gather().flatten().sink_to_list() assert source.loop is client.loop rc = RefCounter() for i in range(3): source.emit(i, metadata=[{'ref': rc}]) while rc.count != 0 and time.monotonic() - start < 2.: time.sleep(1e-2) assert L == [1, 2, 3]