コード例 #1
0
ファイル: core.py プロジェクト: inteplus/mtstreamz
 def __init__(self, upstream, n, **kwargs):
     self.n = n
     self._buffer = []
     self.metadata_buffer = []
     Stream.__init__(self, upstream, **kwargs)
コード例 #2
0
ファイル: test_dataframes.py プロジェクト: salah93/streamz
def stream(request, client):  # flake8: noqa
    if request.param == 'core':
        return Stream()
    else:
        return DaskStream()
 def __init__(self):
     super().__init__()
     self.stream = Stream()
コード例 #4
0
def test_percolate_loop_information(clean):
    source = Stream()
    assert not source.loop
    s = source.timed_window(0.5)
    assert source.loop is s.loop
コード例 #5
0
import json
import time
from uuid import uuid4

from streamz import Stream


def preprocess(x):
    source, _id, data = x
    data["_source"] = source
    data["_id"] = _id
    return json.dumps(data)


sources = ["source-stream-1", "source-stream-2"]
name = str(uuid4())

source = Stream.from_redis_consumer_group(
    sources, "my-group", name, heartbeat_interval=10, claim_timeout=120
)
(
    source.map(preprocess)
    .partition(1000, timeout=3)
    .map(lambda x: "\n".join(x))
    .sink_to_redis_list("preprocessed-data")
)

if __name__ == "__main__":
    source.start()
    time.sleep(5)
コード例 #6
0
def test_zip_latest_reverse():
    a = Stream()
    b = Stream()
    c = a.zip_latest(b)

    L = c.sink_to_list()

    b.emit('a')
    a.emit(1)
    a.emit(2)
    a.emit(3)
    b.emit('b')
    a.emit(4)

    assert L == [(1, 'a'), (2, 'a'), (3, 'a'), (4, 'b')]
コード例 #7
0
def test_disconnect():
    source = Stream()

    upstream = Stream()
    L = upstream.sink_to_list()

    source.emit(1)
    assert L == []
    source.connect(upstream)
    source.emit(2)
    source.emit(3)
    assert L == [2, 3]
    source.disconnect(upstream)
    source.emit(4)
    assert L == [2, 3]
コード例 #8
0
def test_zip():
    a = Stream()
    b = Stream()
    c = sz.zip(a, b)

    L = c.sink_to_list()

    a.emit(1)
    b.emit('a')
    a.emit(2)
    b.emit('b')

    assert L == [(1, 'a'), (2, 'b')]
    d = Stream()
    # test zip from the object itself
    # zip 3 streams together
    e = a.zip(b, d)
    L2 = e.sink_to_list()

    a.emit(1)
    b.emit(2)
    d.emit(3)
    assert L2 == [(1, 2, 3)]
コード例 #9
0
def test_zip_literals():
    a = Stream()
    b = Stream()
    c = sz.zip(a, 123, b)

    L = c.sink_to_list()
    a.emit(1)
    b.emit(2)

    assert L == [(1, 123, 2)]

    a.emit(4)
    b.emit(5)

    assert L == [(1, 123, 2), (4, 123, 5)]
コード例 #10
0
def test_share_common_ioloop(clean):
    a = Stream()
    b = Stream()
    aa = a.timed_window(0.01)
    bb = b.timed_window(0.01)
    assert aa.loop is bb.loop
コード例 #11
0
def test_timed_window_timedelta(clean):
    pytest.importorskip('pandas')
    source = Stream(asynchronous=True)
    a = source.timed_window('10ms')
    assert a.interval == 0.010
コード例 #12
0
def test_accumulate_errors_raises():
    a = Stream()
    b = a.accumulate(lambda x, y: x / y)
    with pytest.raises(ZeroDivisionError):
        a.emit(1)
        a.emit(0)
コード例 #13
0
def test_map_args():
    source = Stream()
    L = source.map(operator.add, 10).sink_to_list()
    source.emit(1)
    assert L == [11]
コード例 #14
0
def test_map_errors_raises():
    a = Stream()
    b = a.map(lambda x: 1 / x)
    with pytest.raises(ZeroDivisionError):
        a.emit(0)
コード例 #15
0
def test_stream_name_str():
    source = Stream(stream_name='this is not a stream')
    assert str(source) == '<this is not a stream; Stream>'
コード例 #16
0
def test_combine_latest_emit_on_stream():
    a = Stream()
    b = Stream()
    c = a.combine_latest(b, emit_on=0)

    L = c.sink_to_list()

    a.emit(1)
    b.emit('a')
    a.emit(2)
    a.emit(3)
    b.emit('b')
    a.emit(4)

    assert L == [(2, 'a'), (3, 'a'), (4, 'b')]
コード例 #17
0
def test_zip_latest():
    a = Stream()
    b = Stream()
    c = a.zip_latest(b)
    d = a.combine_latest(b, emit_on=a)

    L = c.sink_to_list()
    L2 = d.sink_to_list()

    a.emit(1)
    a.emit(2)
    b.emit('a')
    b.emit('b')
    a.emit(3)

    assert L == [(1, 'a'), (2, 'a'), (3, 'b')]
    assert L2 == [(3, 'b')]
コード例 #18
0
def test_no_output():
    source = Stream()
    assert source.emit(1) is None
コード例 #19
0
def test_triple_zip_latest():
    from streamz.core import Stream
    s1 = Stream()
    s2 = Stream()
    s3 = Stream()
    s_simple = s1.zip_latest(s2, s3)
    L_simple = s_simple.sink_to_list()

    s1.emit(1)
    s2.emit('I')
    s2.emit("II")
    s1.emit(2)
    s2.emit("III")
    s3.emit('a')
    s3.emit('b')
    s1.emit(3)
    assert L_simple == [(1, 'III', 'a'), (2, 'III', 'a'), (3, 'III', 'b')]
コード例 #20
0
def test_unique_history():
    source = Stream()
    s = source.unique(history=2)
    L = s.sink_to_list()

    source.emit(1)
    source.emit(2)
    source.emit(1)
    source.emit(2)
    source.emit(1)
    source.emit(2)

    assert L == [1, 2]

    source.emit(3)
    source.emit(2)

    assert L == [1, 2, 3]

    source.emit(1)

    assert L == [1, 2, 3, 1]
コード例 #21
0
def test_docstrings():
    for s in [Stream, Stream()]:
        assert 'every element' in s.map.__doc__
        assert s.map.__name__ == 'map'
        assert 'predicate' in s.filter.__doc__
        assert s.filter.__name__ == 'filter'
コード例 #22
0
def test_union():
    a = Stream()
    b = Stream()
    c = Stream()

    L = a.union(b, c).sink_to_list()

    a.emit(1)
    assert L == [1]
    b.emit(2)
    assert L == [1, 2]
    a.emit(3)
    assert L == [1, 2, 3]
    c.emit(4)
    assert L == [1, 2, 3, 4]
コード例 #23
0
@author: QueenPy
'''
import pandas as pd
from streamz import Stream
from streamz.dataframe import DataFrames
import logging, re
import itertools
# (to do code today on 0224)
# from io import BytesIO, StringIO
# abandon this above IO based tool due to no Path nor Buffer(Memory Register)
# from urllib.parse import urlparse
# import requests

# create streaming object
st = Stream()

# create panda dataframe
df = pd.DataFrame()

# using streaming dataframe


def df2sdf_converter(df):
    sdf = DataFrames(stream=st, example=df)  # stream , example, stream_type
    logging.info('conversion process now')
    show_spec_px = sdf[sdf.item == 'bond'].price.sum()
    print(show_spec_px)


def df2sdf_operator(st):
コード例 #24
0
def test_collect():
    source1 = Stream()
    source2 = Stream()
    collector = source1.collect()
    L = collector.sink_to_list()
    source2.sink(collector.flush)

    source1.emit(1)
    source1.emit(2)
    assert L == []

    source2.emit('anything')  # flushes collector
    assert L == [(1, 2)]

    source2.emit('anything')
    assert L == [(1, 2), ()]

    source1.emit(3)
    assert L == [(1, 2), ()]

    source2.emit('anything')
    assert L == [(1, 2), (), (3, )]
import pandas as pd
import time, sys, asyncio

from streamz import Stream
from streamz.dataframe import DataFrame

import os
basepath = os.path.abspath(os.path.dirname(__file__)) + "/"


def combined_predictions():
    combined_df = pd.read_csv(basepath + 'price_data.csv')
    return combined_df


stream = Stream()
predictions_stream = pd.DataFrame({
    'MSFT_actual': [],
    'MSFT_lstm': [],
    'MSFT_regression': []
})

predictions_streamz_df = DataFrame(stream, example=predictions_stream)
コード例 #26
0
def test_timed_window_str(clean):
    source = Stream()
    s = source.timed_window(.05)
    assert str(s) == '<timed_window: 0.05>'
コード例 #27
0
        return subscription


if __name__=='__main__':
    # Simple example of how these should be used
    # Test with: python -m numismatic.exchanges.luno
    from configparser import ConfigParser
    from pathlib import Path
    import asyncio
    from streamz import Stream
    logging.basicConfig(level=logging.INFO)
    config = ConfigParser()
    config.read(Path.home() / '.coinrc')
    print(list(config.keys()))
    api_key_id = config['Luno'].get('api_key_id', '')
    api_key_secret = config['Luno'].get('api_key_secret', '') 

    output_stream = Stream()
    printer = output_stream.map(print)

    luno = LunoWebsocketClient(output_stream=output_stream, api_key_id=api_key_id,
                            api_key_secret=api_key_secret)
    luno_btc = luno.subscribe('XBTZAR')

    loop = asyncio.get_event_loop()
    future = asyncio.wait([luno_btc], timeout=15)
    completed, pending = loop.run_until_complete(future)
    for task in pending:
        task.cancel()

コード例 #28
0
def test_partition_str():
    source = Stream()
    s = source.partition(2)
    assert str(s) == '<partition: 2>'
コード例 #29
0
from streamz import Stream
from tornado.ioloop import IOLoop

source = Stream(asynchronous=True)
s = source.sliding_window(2).map(sum)
L = s.sink_to_list()  # store result in a list

s.rate_limit(0.5).sink(source.emit)  # pipe output back to input
s.rate_limit(1.0).sink(lambda x: print(L))  # print state of L every second

source.emit(0)  # seed with initial values
source.emit(1)

IOLoop.current().start()
コード例 #30
0
def test_stream_shares_client_loop(loop):  # noqa: F811
    with cluster() as (s, [a, b]):
        with Client(s['address'], loop=loop) as client:  # noqa: F841
            source = Stream()
            d = source.timed_window('20ms').scatter()  # noqa: F841
            assert source.loop is client.loop