def __init__(self, upstream, n, **kwargs):
self.n = n
self._buffer = []
self.metadata_buffer = []
Stream.__init__(self, upstream, **kwargs)
def stream(request, client): # flake8: noqa
if request.param == 'core':
return Stream()
else:
return DaskStream()
def __init__(self):
super().__init__()
self.stream = Stream()
def test_percolate_loop_information(clean):
source = Stream()
assert not source.loop
s = source.timed_window(0.5)
assert source.loop is s.loop
import json
import time
from uuid import uuid4
from streamz import Stream
def preprocess(x):
source, _id, data = x
data["_source"] = source
data["_id"] = _id
return json.dumps(data)
sources = ["source-stream-1", "source-stream-2"]
name = str(uuid4())
source = Stream.from_redis_consumer_group(
sources, "my-group", name, heartbeat_interval=10, claim_timeout=120
)
(
source.map(preprocess)
.partition(1000, timeout=3)
.map(lambda x: "\n".join(x))
.sink_to_redis_list("preprocessed-data")
)
if __name__ == "__main__":
source.start()
time.sleep(5)
def test_zip_latest_reverse():
a = Stream()
b = Stream()
c = a.zip_latest(b)
L = c.sink_to_list()
b.emit('a')
a.emit(1)
a.emit(2)
a.emit(3)
b.emit('b')
a.emit(4)
assert L == [(1, 'a'), (2, 'a'), (3, 'a'), (4, 'b')]
def test_disconnect():
source = Stream()
upstream = Stream()
L = upstream.sink_to_list()
source.emit(1)
assert L == []
source.connect(upstream)
source.emit(2)
source.emit(3)
assert L == [2, 3]
source.disconnect(upstream)
source.emit(4)
assert L == [2, 3]
def test_zip():
a = Stream()
b = Stream()
c = sz.zip(a, b)
L = c.sink_to_list()
a.emit(1)
b.emit('a')
a.emit(2)
b.emit('b')
assert L == [(1, 'a'), (2, 'b')]
d = Stream()
# test zip from the object itself
# zip 3 streams together
e = a.zip(b, d)
L2 = e.sink_to_list()
a.emit(1)
b.emit(2)
d.emit(3)
assert L2 == [(1, 2, 3)]
def test_zip_literals():
a = Stream()
b = Stream()
c = sz.zip(a, 123, b)
L = c.sink_to_list()
a.emit(1)
b.emit(2)
assert L == [(1, 123, 2)]
a.emit(4)
b.emit(5)
assert L == [(1, 123, 2), (4, 123, 5)]
def test_share_common_ioloop(clean):
a = Stream()
b = Stream()
aa = a.timed_window(0.01)
bb = b.timed_window(0.01)
assert aa.loop is bb.loop
def test_timed_window_timedelta(clean):
pytest.importorskip('pandas')
source = Stream(asynchronous=True)
a = source.timed_window('10ms')
assert a.interval == 0.010
def test_accumulate_errors_raises():
a = Stream()
b = a.accumulate(lambda x, y: x / y)
with pytest.raises(ZeroDivisionError):
a.emit(1)
a.emit(0)
def test_map_args():
source = Stream()
L = source.map(operator.add, 10).sink_to_list()
source.emit(1)
assert L == [11]
def test_map_errors_raises():
a = Stream()
b = a.map(lambda x: 1 / x)
with pytest.raises(ZeroDivisionError):
a.emit(0)
def test_stream_name_str():
source = Stream(stream_name='this is not a stream')
assert str(source) == '<this is not a stream; Stream>'
def test_combine_latest_emit_on_stream():
a = Stream()
b = Stream()
c = a.combine_latest(b, emit_on=0)
L = c.sink_to_list()
a.emit(1)
b.emit('a')
a.emit(2)
a.emit(3)
b.emit('b')
a.emit(4)
assert L == [(2, 'a'), (3, 'a'), (4, 'b')]
def test_zip_latest():
a = Stream()
b = Stream()
c = a.zip_latest(b)
d = a.combine_latest(b, emit_on=a)
L = c.sink_to_list()
L2 = d.sink_to_list()
a.emit(1)
a.emit(2)
b.emit('a')
b.emit('b')
a.emit(3)
assert L == [(1, 'a'), (2, 'a'), (3, 'b')]
assert L2 == [(3, 'b')]
def test_no_output():
source = Stream()
assert source.emit(1) is None
def test_triple_zip_latest():
from streamz.core import Stream
s1 = Stream()
s2 = Stream()
s3 = Stream()
s_simple = s1.zip_latest(s2, s3)
L_simple = s_simple.sink_to_list()
s1.emit(1)
s2.emit('I')
s2.emit("II")
s1.emit(2)
s2.emit("III")
s3.emit('a')
s3.emit('b')
s1.emit(3)
assert L_simple == [(1, 'III', 'a'), (2, 'III', 'a'), (3, 'III', 'b')]
def test_unique_history():
source = Stream()
s = source.unique(history=2)
L = s.sink_to_list()
source.emit(1)
source.emit(2)
source.emit(1)
source.emit(2)
source.emit(1)
source.emit(2)
assert L == [1, 2]
source.emit(3)
source.emit(2)
assert L == [1, 2, 3]
source.emit(1)
assert L == [1, 2, 3, 1]
def test_docstrings():
for s in [Stream, Stream()]:
assert 'every element' in s.map.__doc__
assert s.map.__name__ == 'map'
assert 'predicate' in s.filter.__doc__
assert s.filter.__name__ == 'filter'
def test_union():
a = Stream()
b = Stream()
c = Stream()
L = a.union(b, c).sink_to_list()
a.emit(1)
assert L == [1]
b.emit(2)
assert L == [1, 2]
a.emit(3)
assert L == [1, 2, 3]
c.emit(4)
assert L == [1, 2, 3, 4]
@author: QueenPy
'''
import pandas as pd
from streamz import Stream
from streamz.dataframe import DataFrames
import logging, re
import itertools
# (to do code today on 0224)
# from io import BytesIO, StringIO
# abandon this above IO based tool due to no Path nor Buffer(Memory Register)
# from urllib.parse import urlparse
# import requests
# create streaming object
st = Stream()
# create panda dataframe
df = pd.DataFrame()
# using streaming dataframe
def df2sdf_converter(df):
sdf = DataFrames(stream=st, example=df) # stream , example, stream_type
logging.info('conversion process now')
show_spec_px = sdf[sdf.item == 'bond'].price.sum()
print(show_spec_px)
def df2sdf_operator(st):
def test_collect():
source1 = Stream()
source2 = Stream()
collector = source1.collect()
L = collector.sink_to_list()
source2.sink(collector.flush)
source1.emit(1)
source1.emit(2)
assert L == []
source2.emit('anything') # flushes collector
assert L == [(1, 2)]
source2.emit('anything')
assert L == [(1, 2), ()]
source1.emit(3)
assert L == [(1, 2), ()]
source2.emit('anything')
assert L == [(1, 2), (), (3, )]
import pandas as pd
import time, sys, asyncio
from streamz import Stream
from streamz.dataframe import DataFrame
import os
basepath = os.path.abspath(os.path.dirname(__file__)) + "/"
def combined_predictions():
combined_df = pd.read_csv(basepath + 'price_data.csv')
return combined_df
stream = Stream()
predictions_stream = pd.DataFrame({
'MSFT_actual': [],
'MSFT_lstm': [],
'MSFT_regression': []
})
predictions_streamz_df = DataFrame(stream, example=predictions_stream)
def test_timed_window_str(clean):
source = Stream()
s = source.timed_window(.05)
assert str(s) == '<timed_window: 0.05>'
return subscription
if __name__=='__main__':
# Simple example of how these should be used
# Test with: python -m numismatic.exchanges.luno
from configparser import ConfigParser
from pathlib import Path
import asyncio
from streamz import Stream
logging.basicConfig(level=logging.INFO)
config = ConfigParser()
config.read(Path.home() / '.coinrc')
print(list(config.keys()))
api_key_id = config['Luno'].get('api_key_id', '')
api_key_secret = config['Luno'].get('api_key_secret', '')
output_stream = Stream()
printer = output_stream.map(print)
luno = LunoWebsocketClient(output_stream=output_stream, api_key_id=api_key_id,
api_key_secret=api_key_secret)
luno_btc = luno.subscribe('XBTZAR')
loop = asyncio.get_event_loop()
future = asyncio.wait([luno_btc], timeout=15)
completed, pending = loop.run_until_complete(future)
for task in pending:
task.cancel()
def test_partition_str():
source = Stream()
s = source.partition(2)
assert str(s) == '<partition: 2>'
from streamz import Stream from tornado.ioloop import IOLoop source = Stream(asynchronous=True) s = source.sliding_window(2).map(sum) L = s.sink_to_list() # store result in a list s.rate_limit(0.5).sink(source.emit) # pipe output back to input s.rate_limit(1.0).sink(lambda x: print(L)) # print state of L every second source.emit(0) # seed with initial values source.emit(1) IOLoop.current().start()
def test_stream_shares_client_loop(loop): # noqa: F811
with cluster() as (s, [a, b]):
with Client(s['address'], loop=loop) as client: # noqa: F841
source = Stream()
d = source.timed_window('20ms').scatter() # noqa: F841
assert source.loop is client.loop