def test_timed_zip_agents():
scheduler = Stream.scheduler
x = Stream('x')
y = Stream('y')
z = Stream('z')
# timed_zip_agent(in_streams=[x,y], out_stream=z, name='a')
z = timed_zip_f([x, y])
def concat_operator(timed_list):
result = ''
for timestamp_value in timed_list:
result = result + timestamp_value[1]
return result
r = timed_window_f(concat_operator, x, 5, 5)
s = timed_window_f(concat_operator, x, 20, 10)
x.extend([[1, 'a'], [3, 'b'], [10, 'd'], [15, 'e'], [17, 'f']])
y.extend([[2, 'A'], [3, 'B'], [9, 'D'], [20, 'E']])
scheduler.step()
assert z.recent[:z.stop] == \
[[1, ['a', None]], [2, [None, 'A']], [3, ['b', 'B']], [9, [None, 'D']],
[10, ['d', None]], [15, ['e', None]], [17, ['f', None]]]
assert r.recent[:r.stop] == [(5, 'ab'), (15, 'd')]
assert s.recent[:s.stop] == []
x.extend([[21, 'g'], [23, 'h'], [40, 'i'], [55, 'j'], [97, 'k']])
y.extend([[21, 'F'], [23, 'G'], [29, 'H'], [55, 'I']])
scheduler.step()
assert z.recent[:z.stop] == \
[[1, ['a', None]], [2, [None, 'A']], [3, ['b', 'B']], [9, [None, 'D']],
[10, ['d', None]], [15, ['e', None]], [17, ['f', None]],
[20, [None, 'E']], [21, ['g', 'F']], [23, ['h', 'G']],
[29, [None, 'H']], [40, ['i', None]], [55, ['j', 'I']]]
assert r.recent[:r.stop] == [(5, 'ab'), (15, 'd'), (20, 'ef'), (25, 'gh'),
(45, 'i'), (60, 'j')]
assert s.recent[:s.stop] == [(20, 'abdef'), (30, 'defgh'), (40, 'gh'),
(50, 'i'), (60, 'ij'), (70, 'j')]
x.extend([[100, 'l'], [105, 'm']])
y.extend([[100, 'J'], [104, 'K'], [105, 'L'], [107, 'M']])
scheduler.step()
assert z.recent[:z.stop] == \
[[1, ['a', None]], [2, [None, 'A']], [3, ['b', 'B']], [9, [None, 'D']],
[10, ['d', None]], [15, ['e', None]], [17, ['f', None]],
[20, [None, 'E']], [21, ['g', 'F']], [23, ['h', 'G']],
[29, [None, 'H']], [40, ['i', None]], [55, ['j', 'I']],
[97, ['k', None]], [100, ['l', 'J']], [104, [None, 'K']], [105, ['m', 'L']]]
assert r.recent[:r.stop] == [(5, 'ab'), (15, 'd'), (20, 'ef'), (25, 'gh'),
(45, 'i'), (60, 'j'), (100, 'k'), (105, 'l')]
assert s.recent[:s.stop] == [(20, 'abdef'), (30, 'defgh'), (40, 'gh'),
(50, 'i'), (60, 'ij'), (70, 'j'), (100, 'k')]
x.extend([[106, 'n'], [110, 'o']])
scheduler.step()
assert z.recent[:z.stop] == \
[[1, ['a', None]], [2, [None, 'A']], [3, ['b', 'B']], [9, [None, 'D']],
[10, ['d', None]], [15, ['e', None]], [17, ['f', None]],
[20, [None, 'E']], [21, ['g', 'F']], [23, ['h', 'G']],
[29, [None, 'H']], [40, ['i', None]], [55, ['j', 'I']],
[97, ['k', None]], [100, ['l', 'J']], [104, [None, 'K']], [105, ['m', 'L']],
[106, ['n', None]], [107, [None, 'M']]
]
assert r.recent[:r.stop] == [(5, 'ab'), (15, 'd'), (20, 'ef'), (25, 'gh'),
(45, 'i'), (60, 'j'), (100, 'k'), (105, 'l'),
(110, 'mn')]
assert s.recent[:s.stop] == [(20, 'abdef'), (30, 'defgh'), (40, 'gh'),
(50, 'i'), (60, 'ij'), (70, 'j'), (100, 'k'),
(110, 'klmn')]
return
def detect_anomaly(window, short_window_size, threshold, cutoff):
current_average = statistics.mean(window[-short_window_size:])
clipped_window = [v for v in window if v < cutoff]
long_term_average = statistics.mean(clipped_window)
print current_average, long_term_average
anomaly = current_average > long_term_average * threshold
return anomaly
if __name__ == '__main__':
import random
input_sequence = [random.random() for _ in range(100)]
input_sequence.extend([10 + random.random() for _ in range(5)])
input_sequence.extend([random.random() for _ in range(100)])
# Create streams
s = Stream('s')
t = Stream('t')
map_window(
func=detect_anomaly, in_stream=s, out_stream=t,
window_size=50, step_size=1, short_window_size=5,
threshold=3, cutoff=0.9)
print_stream(t)
s.extend(input_sequence)
# Execute a step of the scheduler
Stream.scheduler.step()
def test_merge_agents():
import numpy as np
scheduler = Stream.scheduler
#----------------------------------------------------
# Declare streams
s = Stream('s')
t = Stream('t')
u = Stream('u')
v_stream = Stream('v')
x = Stream('x')
#----------------------------------------------------
# Define functions
def g(lst):
return sum(lst)
def g_args(lst, multiplier):
return sum(lst) * multiplier
def general_f(lst, f):
return f(lst)
def fff(lst, f, addend):
return f(lst, addend)
def hhh(lst, addend):
return sum(lst) + addend
#----------------------------------------------------
# Define agents
d = zip_map(func=g, in_streams=[x, u], out_stream=s, name='d')
zipxu = zip_stream_f([x, u])
zip_map_xu = zip_map_f(sum, [x, u])
zip_map_xu_merge = Stream('zip map xu merge')
zip_map(sum, [x, u], zip_map_xu_merge)
zip_map_g_args = zip_map_f(g_args, [x, u], multiplier=2)
dd = zip_map(func=general_f,
in_streams=[x, u],
out_stream=t,
name='dd',
f=np.mean)
zip_map_ss = zip_map_f(np.mean, [x, u])
dddd = zip_map(func=fff,
in_streams=[x, u],
out_stream=v_stream,
name='dddd',
f=hhh,
addend=10)
#----------------------------------------------------
#----------------------------------------------------
# Append values to stream
x.extend(range(3))
u.extend([10, 15, 18])
scheduler.step()
assert recent_values(s) == [10, 16, 20]
assert recent_values(zip_map_g_args) == [2 * v for v in recent_values(s)]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18)]
assert recent_values(t) == [5, 8, 10]
assert recent_values(zip_map_ss) == [5.0, 8.0, 10.0]
assert recent_values(v_stream) == [20, 26, 30]
assert recent_values(zip_map_xu) == s.recent[:s.stop]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
#----------------------------------------------------
u.append(37)
x.extend(range(3, 5, 1))
scheduler.step()
assert recent_values(s) == [10, 16, 20, 40]
assert recent_values(zip_map_g_args) == [2 * v for v in recent_values(s)]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18), (3, 37)]
assert recent_values(t) == [5, 8, 10, 20]
assert recent_values(v_stream) == [20, 26, 30, 50]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
#----------------------------------------------------
u.extend([96, 95])
scheduler.step()
assert recent_values(s) == [10, 16, 20, 40, 100]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18), (3, 37),
(4, 96)]
assert recent_values(t) == [5, 8, 10, 20, 50]
assert recent_values(v_stream) == [20, 26, 30, 50, 110]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
#----------------------------------------------------
# TEST MERGE_ASYNCH AND MIX
#----------------------------------------------------
x = Stream('x')
y = Stream('y')
z = Stream('z')
w = Stream('w')
def g_asynch(pair):
index, value = pair
if index == 0:
return value * 10
elif index == 1:
return value * 2
else:
raise Exception()
merge_asynch(func=lambda v: v, in_streams=[x, y], out_stream=z)
merge_asynch(func=g_asynch, in_streams=[x, y], out_stream=w)
mix_z = mix_f([x, y])
scheduler.step()
assert recent_values(z) == []
assert recent_values(mix_z) == []
assert recent_values(w) == []
x.append(10)
scheduler.step()
assert recent_values(z) == [(0, 10)]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100]
y.append('A')
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A')]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100, 'AA']
y.append('B')
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A'), (1, 'B')]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100, 'AA', 'BB']
x.append(20)
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A'), (1, 'B'), (0, 20)]
assert recent_values(z) == recent_values(mix_z)
assert recent_values(w) == [100, 'AA', 'BB', 200]
fahrenheit = Stream('fahrenheit')
celsius = Stream('celsius')
def fahrenheit_and_celsius(pair):
index, value = pair
if index == 0:
return (value - 32.0) / 1.8
elif index == 1:
return value
else:
raise Exception()
fahrenheit_stream = Stream('fahrenheit temperatures')
celsius_stream = Stream('celsius temperatures')
centigrade_stream = Stream('centigrade temperatures')
merge_asynch(func=fahrenheit_and_celsius,
in_streams=[fahrenheit_stream, celsius_stream],
out_stream=centigrade_stream)
fahrenheit_stream.append(32)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0]
fahrenheit_stream.append(50)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0]
fahrenheit_stream.append(68)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0, 20.0]
celsius_stream.append(-10.0)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0, 20.0, -10.0]
#----------------------------------------------------
# TEST BLEND
#----------------------------------------------------
x = Stream('x')
y = Stream('y')
z = Stream('z')
def double(v):
return 2 * v
def double_add(v, addend):
return 2 * v + addend
blend(func=double, in_streams=[x, y], out_stream=z)
blend_z = blend_f(double, [x, y])
blend_add_z = blend_f(double_add, [x, y], addend=10)
x.append(1)
scheduler.step()
assert recent_values(z) == [2]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
x.extend(range(2, 4))
scheduler.step()
assert recent_values(z) == [2, 4, 6]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
y.extend(range(100, 102))
scheduler.step()
assert recent_values(z) == [2, 4, 6, 200, 202]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
x.extend([10, 20])
scheduler.step()
assert recent_values(z) == [2, 4, 6, 200, 202, 20, 40]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
#----------------------------------------------------
# TEST MANY
#----------------------------------------------------
# func operates on a list with one element for each input stream.
# func returns a list with one element for each output stream.
def f_many(lst):
return [sum(lst), sum(lst) + 1]
u_stream = Stream(name='u_stream')
v_stream = Stream(name='v_stream')
w_stream = Stream(name='w_stream')
x_stream = Stream(name='x_stream')
multi_agent = multi_element(func=f_many,
in_streams=[u_stream, v_stream],
out_streams=[w_stream, x_stream],
name='multi_agent')
ww_stream, xx_stream = multi_element_f(func=f_many,
in_streams=[u_stream, v_stream],
num_out_streams=2)
u_stream.extend(range(5))
v_stream.extend(range(0, 40, 4))
scheduler.step()
assert recent_values(w_stream) == [0, 5, 10, 15, 20]
assert recent_values(x_stream) == [1, 6, 11, 16, 21]
assert recent_values(ww_stream) == recent_values(w_stream)
assert recent_values(xx_stream) == recent_values(x_stream)
# ------------------------------------
# Test many with args and kwargs
# func operates on a list with one element for each input stream.
# func returns a list with one element for each output stream.
def f_multi_args_kwargs(lst, multiplicand, addend):
return sum(lst) * multiplicand, sum(lst) + addend
u_args_kwargs_stream = Stream(name='u_args_kwargs_stream')
v_args_kwargs_stream = Stream(name='v_args_kwargs_stream')
w_args_kwargs_stream = Stream(name='w_args_kwargs_stream')
x_args_kwargs_stream = Stream(name='x_args_kwargs_stream')
multi_args_kwargs_agent = multi_element(
func=f_multi_args_kwargs,
in_streams=[u_args_kwargs_stream, v_args_kwargs_stream],
out_streams=[w_args_kwargs_stream, x_args_kwargs_stream],
name='multi_args_kwargs_agent',
multiplicand=2,
addend=10)
ww_args_kwargs_stream, xx_args_kwargs_stream = multi_element_f(
func=f_multi_args_kwargs,
in_streams=[u_args_kwargs_stream, v_args_kwargs_stream],
num_out_streams=2,
multiplicand=2,
addend=10)
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.extend(range(5))
v_args_kwargs_stream.extend(range(0, 40, 4))
scheduler.step()
assert recent_values(w_args_kwargs_stream) == [0, 10, 20, 30, 40]
assert recent_values(x_args_kwargs_stream) == [10, 15, 20, 25, 30]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.append(100)
v_args_kwargs_stream.extend(range(40, 80, 4))
scheduler.step()
assert recent_values(w_args_kwargs_stream) == \
[0, 10, 20, 30, 40, 240]
assert recent_values(x_args_kwargs_stream) == \
[10, 15, 20, 25, 30, 130]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.extend([200, 300])
scheduler.step()
v_args_kwargs_stream.append(100)
scheduler.step()
assert recent_values(w_args_kwargs_stream) == \
[0, 10, 20, 30, 40, 240, 448, 656]
assert recent_values(x_args_kwargs_stream) == \
[10, 15, 20, 25, 30, 130, 234, 338]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
#----------------------------------------------------
#----------------------------------------------------
# TEST STREAM ARRAY
#----------------------------------------------------
#----------------------------------------------------
#----------------------------------------------------
# Test zip_map with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
z = StreamArray('z', dimension=2)
a = StreamArray('a')
zip_map(func=lambda v: v, in_streams=[x, y], out_stream=z)
zip_map(func=np.mean, in_streams=[x, y], out_stream=a)
x.extend(np.linspace(0.0, 9.0, 10))
scheduler.step()
y.extend(np.linspace(0.0, 4.0, 5))
scheduler.step()
expected_array = np.vstack(
[np.linspace(0.0, 4.0, 5),
np.linspace(0.0, 4.0, 5)]).T
assert np.array_equal(recent_values(z), expected_array)
expected_means = np.linspace(0.0, 4.0, 5)
assert np.array_equal(recent_values(a), expected_means)
#----------------------------------------------------
# Test blend with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
z = StreamArray('z')
a = StreamArray('a')
def double(v):
return 2 * v
def double_add(v, addend):
return 2 * v + addend
blend(func=double, in_streams=[x, y], out_stream=z)
blend(func=double_add, in_streams=[x, y], out_stream=a, addend=10.0)
x.append(np.array(1.0))
scheduler.step()
assert np.array_equal(recent_values(z), np.array([2.0]))
assert np.array_equal(recent_values(a), recent_values(z) + 10.0)
x.extend(np.linspace(2.0, 3.0, 2))
scheduler.step()
assert np.array_equal(recent_values(z), np.array([2., 4., 6.]))
assert np.array_equal(recent_values(a), recent_values(z) + 10.0)
y.extend(np.linspace(100.0, 101.0, 2))
scheduler.step()
assert np.array_equal(recent_values(z), [2., 4., 6., 200., 202.])
assert np.array_equal(recent_values(a), recent_values(z) + 10.0)
x.extend([10., 20.])
scheduler.step()
assert np.array_equal(recent_values(z), [2., 4., 6., 200., 202., 20., 40.])
assert np.array_equal(recent_values(a), recent_values(z) + 10.0)
#----------------------------------------------------
# Test merge_asynch with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
dt_0 = np.dtype([('time', int), ('value', float)])
z = StreamArray('z', dimension=2)
merge_asynch(func=lambda v: v, in_streams=[x, y], out_stream=z)
scheduler.step()
assert np.array_equal(recent_values(z), np.empty(shape=(0, 2)))
x.append(np.array(10.0))
scheduler.step()
assert np.array_equal(recent_values(z), np.array([(0, 10.0)]))
y.append(np.array(1.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.0)])
y.append(np.array(2.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.0), (1, 2.0)])
x.append(np.array(20.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.), (1, 2.),
(0, 20.)])
#----------------------------------------------------------------
# Test window merge agent with no state
r = Stream('r')
w = Stream('w')
x = Stream('x')
a = Stream('a')
def h(list_of_windows):
return sum([sum(window) for window in list_of_windows])
merge_window(func=h,
in_streams=[r, w],
out_stream=x,
window_size=3,
step_size=3)
merge_stream = merge_window_f(func=h,
in_streams=[r, w],
window_size=3,
step_size=3)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window merge agent with state
def h_with_state(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_with_state,
in_streams=[r, w],
out_stream=a,
window_size=3,
step_size=3,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
r.extend(range(16))
scheduler.step()
assert recent_values(r) == range(16)
assert recent_values(x) == []
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(a) == []
w.extend([10, 12, 14, 16, 18])
scheduler.step()
assert recent_values(r) == range(16)
assert recent_values(w) == [10, 12, 14, 16, 18]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(a) == [39]
#----------------------------------------------------------------
r.extend([10, -10, 21, -20])
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(a) == [39]
#----------------------------------------------------------------
w.append(20)
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(a) == [39, 67]
#----------------------------------------------------------------
r.extend([-1, 1, 0])
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(a) == [39, 67]
#----------------------------------------------------------------
# TEST MERGE_WINDOW WITH STREAM ARRAY
#----------------------------------------------------------------
x = StreamArray('x', dimension=2)
b = StreamArray('b', dimension=2)
a = StreamArray('a', dimension=2)
#----------------------------------------------------------------
# Test window merge agent with state
def h_array(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_array,
in_streams=[x, a],
out_stream=b,
window_size=2,
step_size=2,
state=0)
#----------------------------------------------------------------
x.extend(np.array([[1., 5.], [7., 11.]]))
a.extend(np.array([[0., 1.], [2., 3.]]))
scheduler.step()
np.array_equal(recent_values(b), np.empty(shape=(0, 2)))
a.extend(np.array([[0., 1.], [1., 0.]]))
scheduler.step()
np.array_equal(recent_values(b), np.empty(shape=(0, 2)))
x.extend(np.array([[14., 18.], [18., 30.], [30., 38.], [34., 42.]]))
scheduler.step()
#-------------------------------------------------------------------
# TEST MERGE_LIST
#-------------------------------------------------------------------
# Function g operates on a list of lists, one list for each input
# stream, to return a single list for the output stream.
x = Stream('x list merge')
u = Stream('u list merge')
s = Stream('s list merge')
def g(list_of_lists):
return [sum(snapshot) for snapshot in zip(*list_of_lists)]
d = merge_list(func=g, in_streams=[x, u], out_stream=s, name='d')
ss = merge_list_f(g, [x, u])
x.extend(range(4))
u.extend(range(10, 20, 2))
scheduler.step()
assert recent_values(x) == [0, 1, 2, 3]
assert recent_values(u) == [10, 12, 14, 16, 18]
assert recent_values(s) == [10, 13, 16, 19]
return
