def test_filter():
source = Stream()
L = source.filter(lambda x: x % 2 == 0).sink_to_list()
for i in range(10):
source.emit(i)
assert L == [0, 2, 4, 6, 8]
def test_zip_same():
a = Stream()
b = a.zip(a)
L = b.sink_to_list()
a.emit(1)
a.emit(2)
assert L == [(1, 1), (2, 2)]
def test_filter_none():
source = Stream()
L = source.filter(None).sink_to_list()
for i in range(10):
source.emit(i % 3)
assert L == [1, 2, 1, 2, 1, 2]
def test_zip_latest():
a = Stream()
b = Stream()
c = a.zip_latest(b)
d = a.combine_latest(b, emit_on=a)
L = c.sink_to_list()
L2 = d.sink_to_list()
a.emit(1)
a.emit(2)
b.emit('a')
b.emit('b')
a.emit(3)
assert L == [(1, 'a'), (2, 'a'), (3, 'b')]
assert L2 == [(3, 'b')]
def test_triple_zip_latest():
from streamz.core import Stream
s1 = Stream()
s2 = Stream()
s3 = Stream()
s_simple = s1.zip_latest(s2, s3)
L_simple = s_simple.sink_to_list()
s1.emit(1)
s2.emit('I')
s2.emit("II")
s1.emit(2)
s2.emit("III")
s3.emit('a')
s3.emit('b')
s1.emit(3)
assert L_simple == [(1, 'III', 'a'), (2, 'III', 'a'), (3, 'III', 'b')]
def test_partition():
source = Stream()
L = source.partition(2).sink_to_list()
for i in range(10):
source.emit(i)
assert L == [(0, 1), (2, 3), (4, 5), (6, 7), (8, 9)]
def test_remove():
source = Stream()
L = source.remove(lambda x: x % 2 == 0).sink_to_list()
for i in range(10):
source.emit(i)
assert L == [1, 3, 5, 7, 9]
def test_separate_thread_without_time(loop, thread):
assert thread.is_alive()
source = Stream(loop=loop)
L = source.map(inc).sink_to_list()
for i in range(10):
source.emit(i)
assert L[-1] == i + 1
def test_unique():
source = Stream()
L = source.unique().sink_to_list()
source.emit(1)
source.emit(2)
source.emit(1)
assert L == [1, 2]
def test_sliding_window():
source = Stream()
L = source.sliding_window(2).sink_to_list()
for i in range(10):
source.emit(i)
assert L == [(0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7),
(7, 8), (8, 9)]
def test_concat():
source = Stream()
L = source.concat().sink_to_list()
source.emit([1, 2, 3])
source.emit([4, 5])
source.emit([6, 7, 8])
assert L == [1, 2, 3, 4, 5, 6, 7, 8]
def test_pluck():
a = Stream()
L = a.pluck(1).sink_to_list()
a.emit([1, 2, 3])
assert L == [2]
a.emit([4, 5, 6, 7, 8, 9])
assert L == [2, 5]
with pytest.raises(IndexError):
a.emit([1])
def test_zip_timeout():
a = Stream()
b = Stream()
c = sz.zip(a, b, maxsize=2)
L = c.sink_to_list()
a.emit(1)
a.emit(2)
future = a.emit(3)
with pytest.raises(gen.TimeoutError):
yield gen.with_timeout(timedelta(seconds=0.01), future)
b.emit('a')
yield future
assert L == [(1, 'a')]
def test_frequencies():
source = Stream()
L = source.frequencies().sink_to_list()
source.emit('a')
source.emit('b')
source.emit('a')
assert L[-1] == {'a': 2, 'b': 1}
def test_map_errors_log():
a = Stream(asynchronous=True)
b = a.delay(0.001).map(lambda x: 1 / x) # noqa: F841
with captured_logger('streamz') as logger:
a._emit(0)
yield gen.sleep(0.1)
out = logger.getvalue()
assert 'ZeroDivisionError' in out
def test_zip_metadata():
a = Stream()
b = Stream()
L = metadata(a.zip(b)).sink_to_list()
a.emit(1, metadata=[{'v': 1}])
b.emit(2, metadata=[{'v': 2}])
a.emit(3)
b.emit(4, metadata=[{'v': 4}])
assert L == [
[{
'v': 1
}, {
'v': 2
}], # first emit when 2 is introduced
[{
'v': 4
}] # second emit when 4 is introduced, and 3 has no metadata
]
def test_partition_timeout():
source = Stream()
L = source.partition(10, timeout=0.01).sink_to_list()
for i in range(5):
source.emit(i)
sleep(0.1)
assert L == [(0, 1, 2, 3, 4)]
def test_rate_limit():
source = Stream()
L = source.rate_limit(0.05).sink_to_list()
start = time()
for i in range(5):
yield source.emit(i)
stop = time()
assert stop - start > 0.2
assert len(L) == 5
def test_starmap():
def add(x=0, y=0):
return x + y
source = Stream()
L = source.starmap(add).sink_to_list()
source.emit((1, 10))
assert L[0] == 11
def test_pluck_list():
a = Stream()
L = a.pluck([0, 2]).sink_to_list()
a.emit([1, 2, 3])
assert L == [(1, 3)]
a.emit([4, 5, 6, 7, 8, 9])
assert L == [(1, 3), (4, 6)]
with pytest.raises(IndexError):
a.emit([1])
def test_map():
def add(x=0, y=0):
return x + y
source = Stream()
L = source.map(add, y=10).sink_to_list()
source.emit(1)
assert L[0] == 11
def test_accumulate_errors_log():
a = Stream()
b = a.delay(0.001).accumulate(lambda x, y: x / y)
with captured_logger('streamz') as logger:
a._emit(1)
a._emit(0)
yield gen.sleep(0.1)
out = logger.getvalue()
assert 'ZeroDivisionError' in out
def test_connect():
source_downstream = Stream()
# connect assumes this default behaviour
# of stream initialization
assert source_downstream.parents == []
assert source_downstream.children == [None]
# initialize the second stream to connect to
source_upstream = Stream()
sout = source_downstream.map(lambda x: x + 1)
L = list()
sout = sout.map(L.append)
source_upstream.connect(source_downstream)
source_upstream.emit(2)
source_upstream.emit(4)
assert L == [3, 5]
def test_accumulate_errors_log():
a = Stream(asynchronous=True)
b = a.delay(0.001).accumulate(lambda x, y: x / y,
with_state=True) # noqa: F841
with captured_logger('streamz') as logger:
a._emit(1)
a._emit(0)
yield gen.sleep(0.1)
out = logger.getvalue()
assert 'ZeroDivisionError' in out
def test_unique_key():
source = Stream()
L = source.unique(key=lambda x: x % 2, history=1).sink_to_list()
source.emit(1)
source.emit(2)
source.emit(4)
source.emit(6)
source.emit(3)
assert L == [1, 2, 3]
def test_partition_ref_counts():
source = Stream()
_ = source.partition(2)
for i in range(10):
r = RefCounter()
source.emit(i, metadata=[{'ref': r}])
if i % 2 == 0:
assert r.count == 1
else:
assert r.count == 0
def test_multi_connect():
source0 = Stream()
source1 = Stream()
source_downstream = source0.union(source1)
# connect assumes this default behaviour
# of stream initialization
assert not source_downstream.downstreams
# initialize the second stream to connect to
source_upstream = Stream()
sout = source_downstream.map(lambda x: x + 1)
L = list()
sout = sout.map(L.append)
source_upstream.connect(source_downstream)
source_upstream.emit(2)
source_upstream.emit(4)
assert L == [3, 5]
def test_combine_latest_ref_counts():
a = Stream()
b = Stream()
_ = a.combine_latest(b)
ref1 = RefCounter()
a.emit(1, metadata=[{'ref': ref1}])
assert ref1.count == 1
# The new value kicks out the old value
ref2 = RefCounter()
a.emit(2, metadata=[{'ref': ref2}])
assert ref1.count == 0
assert ref2.count == 1
# The value on stream a is still retained and the value on stream b is new
ref3 = RefCounter()
b.emit(3, metadata=[{'ref': ref3}])
assert ref2.count == 1
assert ref3.count == 1
def test_sink_to_file():
with tmpfile() as fn:
source = Stream()
with sink_to_file(fn, source) as f:
source.emit('a')
source.emit('b')
with open(fn) as f:
data = f.read()
assert data == 'a\nb\n'
def test_kwargs():
source = Stream()
def f(acc, x, y=None):
acc = acc + x + y
return acc
L = source.scan(f, y=10).sink_to_list()
for i in range(3):
source.emit(i)
assert L == [0, 11, 23]
