def test_if(self):
import tributary.streaming as ts
assert ts.run(
ts.Print(
ts.If(ts.Foo(conditionals), ts.Foo(if_stream),
ts.Foo(else_stream)))) == [1, 2, 3]
def test_reduce_stateful(self):
def foo1():
yield 0
yield 2
yield 4
def foo2():
yield 1
yield 3
yield 5
def foo3():
yield 2
yield 4
yield 6
def reduce(node1_value, node2_value, node3_value, reducer_node):
if not hasattr(reducer_node, "state"):
# on first call, make sure node tracks state
reducer_node.set("state", {"n1": None, "n2": None, "n3": None})
if node1_value is not None:
reducer_node.state["n1"] = node1_value
if node2_value is not None:
reducer_node.state["n2"] = node2_value
if node3_value is not None:
reducer_node.state["n3"] = node3_value
return reducer_node.state
n1 = ts.Foo(foo1)
n2 = ts.Foo(foo2)
n3 = ts.Foo(foo3)
r = ts.Reduce(n1, n2, n3, reducer=reduce, inject_node=True)
out = ts.run(r)
print(out)
assert out == [
{
"n1": 0,
"n2": 1,
"n3": 2
},
{
"n1": 2,
"n2": 3,
"n3": 4
},
{
"n1": 4,
"n2": 5,
"n3": 6
},
]
def test_merge(self):
def foo1():
yield 1
yield 3
def foo2():
yield 2
yield 4
yield 6
out = ts.Merge(ts.Print(ts.Foo(foo1)), ts.Print(ts.Foo(foo2)))
assert ts.run(out) == [(1, 2), (3, 4)]
def test_run_async_generator(self):
async def foo():
yield 1
yield 2
t = ts.Foo(foo)
assert ts.run(t) == [1, 2]
def test_construct_streaming(self):
# adapted from https://gist.github.com/raddy/bd0e977dc8437a4f8276
# spot, strike, vol, days till expiry, interest rate, call or put (1,-1)
spot, strike, vol, dte, rate, cp = sy.symbols('spot strike vol dte rate cp')
T = dte / 260.
N = syNormal('N', 0.0, 1.0)
d1 = (sy.ln(spot / strike) + (0.5 * vol ** 2) * T) / (vol * sy.sqrt(T))
d2 = d1 - vol * sy.sqrt(T)
TimeValueExpr = sy.exp(-rate * T) * (cp * spot * cdf(N)(cp * d1) - cp * strike * cdf(N)(cp * d2))
PriceClass = ts.construct_streaming(TimeValueExpr)
def strikes():
strike = 205
while strike <= 220:
yield strike
strike += 2.5
price = PriceClass(spot=tss.Const(210.59),
# strike=tss.Print(tss.Const(205), text='strike'),
strike=tss.Foo(strikes, interval=1),
vol=tss.Const(14.04),
dte=tss.Const(4),
rate=tss.Const(.2175),
cp=tss.Const(-1))
ret = tss.run(tss.Print(price._node))
time.sleep(2)
assert len(ret) == 7
def test_fixed_map(self):
def foo():
yield [1, 2, 3]
yield [4, 5, 6]
out = ts.Reduce(*[x + 1 for x in ts.Foo(foo).map(3)])
assert ts.run(out) == [(2, 3, 4), (5, 6, 7)]
def test_run_async_foo(self):
async def foo():
await asyncio.sleep(0.1)
return 5
t = ts.Foo(foo, count=1)
assert ts.run(t) == [5]
def test_reduce(self):
def foo1():
yield 1
yield 4
def foo2():
yield 2
yield 5
yield 7
def foo3():
yield 3
yield 6
yield 8
out = ts.Reduce(ts.Foo(foo1), ts.Foo(foo2), ts.Foo(foo3))
assert ts.run(out) == [(1, 2, 3), (4, 5, 6)]
def test_diff(self):
ret = ts.run(ts.Diff(ts.Foo(foo2)))
vals = [1, 2, 0, 5, 4]
comp = [None] + [vals[i] - vals[i - 1] for i in range(1, 5)]
assert ret[0] is None
for i, x in enumerate(ret[1:]):
assert (x - comp[i + 1]) < 0.001
def test_http(self):
'''Test http server'''
def foo():
yield 'x'
yield 'y'
yield 'z'
out = ts.HTTPSink(ts.Foo(foo), url='http://localhost:8080')
assert len(ts.run(out)) == 3
def test_socketio(self):
'''Test socketio streaming'''
def foo():
yield 'a'
yield 'b'
yield 'c'
out = ts.SocketIOSink(ts.Foo(foo), url='http://localhost:8069')
assert ts.run(out) == ['a', 'b', 'c']
def test_http(self):
"""Test http server"""
def foo():
yield "x"
yield "y"
yield "z"
out = ts.HTTPSink(ts.Foo(foo), url="http://localhost:8080")
assert len(ts.run(out)) == 3
def test_websocket_server(self):
"""Test websocket server"""
def foo():
yield "x"
yield "y"
yield "z"
out = ts.WebSocketServerSink(ts.Foo(foo), port=1234)
assert len(ts.run(out)) == 3
def test_websocket(self):
'''Test websocket streaming'''
def foo():
yield 'x'
yield 'y'
yield 'z'
out = ts.WebSocketSink(ts.Foo(foo), url='ws://localhost:8080', response=True)
assert len(ts.run(out)) == 3
def test_socketio(self):
"""Test socketio streaming"""
def foo():
yield "a"
yield "b"
yield "c"
out = ts.SocketIOSink(ts.Foo(foo), url="http://localhost:8069")
assert ts.run(out) == ["a", "b", "c"]
def test_interval(self):
def reducer(short, long, node):
if not node.has("state"):
node.set("state", {"long": 0, "short": 0})
if long:
node.state["long"] += long
if short:
node.state["short"] += short
return node.state.copy()
async def short():
await sleep(1)
return 1
async def long():
await sleep(2)
return 2
def interval(foo, time=1):
task = None
async def _ret():
nonlocal task
if task is None:
task = asyncio.ensure_future(foo())
if not task.done():
await sleep(time)
return None
result = task.result()
task = None
return result
return _ret
short_node = ts.Foo(interval(short, 1), count=5)
long_node = ts.Foo(interval(long, 1), count=5)
out = ts.Reduce(short_node,
long_node,
reducer=reducer,
inject_node=True).print()
ts.run(out)
def test_foo_generator(self):
def foo():
yield 1
yield 2
yield 3
yield 4
yield 5
t = ts.Foo(foo)
assert ts.run(t) == [1, 2, 3, 4, 5]
def test_websocket(self):
"""Test websocket streaming"""
def foo():
yield "x"
yield "y"
yield "z"
out = ts.WebSocketSink(ts.Foo(foo),
url="ws://localhost:8080",
response=True)
assert len(ts.run(out)) == 3
def test_kafka(self):
"""Test streaming with Kafka"""
def foo():
yield "a"
yield "b"
yield "c"
out = ts.KafkaSink(ts.Foo(foo),
servers="localhost:9092",
topic="tributary")
assert ts.run(out) == ["a", "b", "c"]
def test_kafka(self):
'''Test streaming with Kafka'''
def foo():
yield 'a'
yield 'b'
yield 'c'
out = ts.KafkaSink(ts.Foo(foo),
servers='localhost:9092',
topic='tributary')
assert ts.run(out) == ['a', 'b', 'c']
def test_file(self):
file = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'test_file_data.csv'))
def foo():
yield 1
yield 2
yield 3
yield 4
out = ts.FileSink(ts.Foo(foo), filename=file)
assert ts.run(out) == [1, 2, 3, 4]
def test_run_stop(self):
import time
import tributary.streaming as ts
async def foo():
while True:
yield 1
await asyncio.sleep(1)
g = ts.run(ts.Print(ts.Foo(foo)), blocking=False)
time.sleep(5)
g.stop()
def test_function(self):
def foo(*args):
for _ in range(5):
yield _
lazy_node = tl.Node(callable=foo) + 5
# 5 6 7 8 9
streaming_node = ts.Print(ts.Foo(foo) - 5, 'streaming:')
# -5 -4 -3 -2 -1
out = ts.Print(t.LazyToStreaming(lazy_node), 'lazy:') + streaming_node
x = ts.run(out)
# 0 2 4 6 8
print(x)
assert x == [0, 2, 4, 6, 8]
def test_function_order(self):
def foo(*args):
for _ in range(5):
yield _
lazy_node = tl.Node(callable=foo) + 5
# 5 6 7 8 9
streaming_node = ts.Print(ts.Foo(foo) - 5, "streaming:")
# -5 -4 -3 -2 -1
out = streaming_node + ts.Print(lazy_node, "lazy:")
x = ts.run(out)
# 0 2 4 6 8
print(x)
assert x == [0, 2, 4, 6, 8]
def test_value(self):
def foo(*args):
for _ in range(5):
lazy_node.setValue(_ + 1)
yield _
lazy_node = tl.Node(value=0)
# 5 6 7 8 9
streaming_node = ts.Print(ts.Foo(foo) - 5, "streaming:")
# -5 -4 -3 -2 -1
out = ts.Print(t.LazyToStreaming(lazy_node) + 5,
"lazy:") + streaming_node
x = ts.run(out)
# 0 2 4 6 8
print(x)
assert x == [0, 2, 4, 6, 8]
def test_value_order(self):
lazy_node = tl.Node(value=0)
# 5 6 7 8 9
def foo(lazy_node=lazy_node):
for _ in range(5):
yield _
lazy_node.setValue(_ + 1)
streaming_node = ts.Print(ts.Foo(foo) - 5, "streaming:")
# -5 -4 -3 -2 -1
out = streaming_node + ts.Print(lazy_node + 5, "lazy:")
x = ts.run(out)
# 0 2 4 6 8
print(x)
assert x == [0, 2, 4, 6, 8]
def test_http(self):
def foo():
yield 1
yield 2
yield 3
def parser(data):
return ["INSERT INTO test(col1) VALUES ({});".format(data)]
query = ['SELECT * FROM test']
out = ts.PostgresSink(ts.Foo(foo),
query_parser=parser,
user='******',
database='postgres',
password='******',
host='localhost:5432')
assert len(ts.run(out)) == 3
def test_pg(self):
def foo():
yield 1
yield 2
yield 3
def parser(data):
return ["INSERT INTO test(col1) VALUES ({});".format(data)]
out = ts.PostgresSink(
ts.Foo(foo),
query_parser=parser,
user="******",
database="postgres",
password="******",
host="localhost:5432",
)
assert len(ts.run(out)) == 3
def test_file(self):
file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'test_file_data.csv'))
if os.path.exists(file):
os.remove(file)
def foo():
yield 1
yield 2
yield 3
yield 4
def read_file(file):
with open(file, 'r') as fp:
data = fp.read()
return [int(x) for x in data]
# Test that output is equal to what is read (generalized)
out = ts.FileSink(ts.Foo(foo), filename=file)
assert ts.run(out) == read_file(file)
def test_process(self):
def foo():
yield {"a": 1, "b": 2}
yield {"a": 2, "b": 4}
yield {"a": 3, "b": 6}
yield {"a": 4, "b": 8}
def _json(val):
return JSON.dumps(val)
cmd = "{} {} --1".format(
sys.executable, os.path.join(os.path.dirname(__file__), "echo.py"))
print(cmd)
ret = ts.run(
ts.Subprocess(ts.Foo(foo).print("in:"), cmd,
json=True).print("out:"))
print(ret)
assert ret == [
{
"a": 1,
"b": 2
},
{
"a": 2,
"b": 4
},
{
"a": 3,
"b": 6
},
{
"a": 4,
"b": 8
},
]
