def test_subtract_mean():
x = StreamArray('x', dtype=float)
y = StreamArray('x', dtype=float)
subtract_mean(x, window_size=2, step_size=1, y=y)
x.extend(np.arange(5, dtype=float))
run()
print(recent_values(y))
def test_incremental_PCA():
# GET DATA
iris = load_iris()
raw_data = iris.data
target = iris.target
n_samples, n_features = raw_data.shape
# RANDOMIZE DATA
n_components = 2
perm = np.random.permutation(n_samples)
raw_data = raw_data[perm]
target = target[perm]
# SET UP PLOT
plotter = plot_incremental(target=target,
colors=['navy', 'turquoise', 'darkorange'],
labels=iris.target_names)
# RUN ALGORITHM
in_stream = StreamArray(dimension=n_features, dtype=float)
out_stream = StreamArray(dimension=n_components, dtype=float)
incremental_PCA(in_stream,
out_stream,
n_components,
batch_size=30,
n_recompute=5,
plotter=plotter)
in_stream.extend(raw_data)
run()
print(out_stream.recent[:out_stream.stop])
def test_minus_operator_with_arrays():
x = StreamArray(dtype=int)
y = StreamArray(dtype=int)
z = y - x
x.extend(np.arange(3))
y.extend(np.arange(100, 105, 2))
run()
assert np.array_equal(recent_values(z), np.array([
100, 101, 102]))
def test_map_list_with_arrays():
from op import map_list
x = StreamArray(dtype=int)
y = StreamArray(dtype=int)
def f(A): return 2*A
map_list(f, x, y)
x.extend(np.arange(5))
run()
assert np.array_equal(recent_values(y), 2*np.arange(5))
def pitchshift_stream(sound_array, n, window_size=2**13, h=2**11):
"""
Changes the pitch of a sound by n semitones.
Notes
-----
This application has 2 sink_window agents and 3 streams x, y, z.
Stretch agent: The first agent gets input x and outputs y which
stretches the data in stream x. The stretching code is from Zulko,
pianoputer.
Speed up agent: The next agent gets input y and outputs z which
speeds up y by the specified factor. This agent interpolates the
data in y to the number of points determined by factor.
"""
factor = 2**(1.0 * n / 12.0)
f = 1.0/factor
# Declare streams
x = StreamArray('x', dtype=np.int16)
y = StreamArray('y', dtype=np.int16)
z = StreamArray('z', dtype=np.int16)
# Define the stretch agent
stretch_object = Stretch(
in_stream=x, out_stream=y, factor=factor,
window_size=window_size, h=h)
sink_window(
func=stretch_object.stretch, in_stream=x,
window_size=window_size+h, step_size=int(h*f))
# Define the speedup agent.
def f(window, out_stream):
indices = np.arange(0, window_size, factor)
out_stream.extend(
np.int16(np.interp(
indices, np.arange(window_size), window)))
sink_window(func=f, in_stream=y, window_size=window_size,
step_size=window_size, out_stream=z)
# Partition sound_array into sound bites. Extend the
# input with a sequence of sound bites and run each
# sound bite until the sound_array data is finished.
sound_bite_size = 2**14
for i in range(0, sound_array.size, sound_bite_size):
# sound_bite = sound_array[i:i+sound_bite_size]
x.extend(sound_array[i:i+sound_bite_size])
run()
# Process any data in sound_array that wasn't processed
# in the for loop.
x.extend(sound_array[i:])
# Return the result.
return z.recent[:z.stop]
def test_sink_3():
x = StreamArray(dtype='int')
y = StreamArray()
@sink_w
def f(window, y):
y.append(window[-1] - np.mean(window))
f(x, window_size=2, step_size=1, y=y)
x.extend(np.arange(5))
run()
assert (np.array_equal
(recent_values(y), np.array([0.5, 0.5, 0.5, 0.5])))
def test_minus_operator_with_arrays_and_dimension():
x = StreamArray(dimension=3, dtype=int)
y = StreamArray(dimension=3, dtype=int)
z = y - x
A = np.array([[10, 20, 30], [40, 50, 60]])
B= np.array([[100, 100, 100], [200, 200, 200], [300, 300, 00]])
x.extend(A)
y.extend(B)
run()
assert np.array_equal(recent_values(z), np.array([
[ 90, 80, 70],
[160, 150, 140]]))
def test_iot():
x = StreamArray(dtype=int)
y = StreamArray(dtype=int)
z = StreamArray(dtype=int)
u = StreamArray(dtype=int)
v = StreamArray(dtype=int)
def f(A, y, z):
"""
Function wrapped by an iot agent. The first parameter
'A' is an array obtained from the input stream of the
agent. The other parameters, y and z, are positional
arguments (*args) of the function.
The function returns a pointer into the input stream.
In this example, each call to the function processes
the entire input array 'A' and so the function returns
len(A).
"""
y.extend(2 * A)
z.extend(3 * A)
# Return a pointer into the input array.
return len(A)
def g(A, u, v):
"""
Parameters are similar to f.
"""
u.extend(A + A)
v.extend(A**2)
return len(A)
# Create agents that wrap functions f and g.
iot(f, x, y, z)
iot(g, x, u, v)
# Extend stream x with an array
x.extend(np.arange(5, dtype=int))
run()
assert np.array_equal(recent_values(y), 2 * np.arange(5, dtype=int))
assert np.array_equal(recent_values(z), 3 * np.arange(5, dtype=int))
assert np.array_equal(recent_values(u), 2 * np.arange(5, dtype=int))
assert np.array_equal(recent_values(v), np.arange(5, dtype=int)**2)
# Extend stream x with another array
x.extend(np.arange(5, 10, dtype=int))
run()
assert np.array_equal(recent_values(y), 2 * np.arange(10, dtype=int))
assert np.array_equal(recent_values(z), 3 * np.arange(10, dtype=int))
assert np.array_equal(recent_values(u), 2 * np.arange(10, dtype=int))
assert np.array_equal(recent_values(v), np.arange(10, dtype=int)**2)
def test_iot_class():
x = StreamArray(name='x', dtype=int)
y = StreamArray(name='y', dtype=int)
# Create an agent that wraps np.sum
sw = sliding_window_test(
func=np.sum, in_stream=x, out_stream=y, window_size=5, step_size=2)
x.extend(np.arange(10, dtype=int))
run()
assert np.array_equal(recent_values(y), np.array([10, 20, 30]))
x.extend(np.arange(10, 20, dtype=int))
run()
assert np.array_equal(recent_values(y),
np.array([10, 20., 30, 40, 50, 60, 70, 80]))
def test_iot_class():
x = StreamArray(dtype=int)
y = StreamArray(dtype=float)
sw = sliding_window_test(func=np.sum,
in_stream=x,
out_stream=y,
window_size=5,
step_size=2)
x.extend(np.arange(10, dtype=int))
run()
assert np.array_equal(recent_values(y), np.array([10., 20., 30.]))
x.extend(np.arange(10, 20, dtype=int))
run()
assert np.array_equal(recent_values(y),
np.array([10., 20., 30., 40., 50., 60., 70., 80.]))
def test_kmeans_sliding_windows():
print '-----------------------------------------'
print
print 'testing kmeans sliding windows'
num_dimensions = 2
window_size = 12
step_size = 2
num_clusters = 4
in_stream = StreamArray(name='in', dimension=num_dimensions)
out_stream = StreamArray(name='out', dimension=window_size, dtype=int)
kmeans_sliding_windows(in_stream, out_stream, window_size, step_size,
num_clusters)
points = np.array([
[+1.0, +1.0],
[+1.1, +1.1],
[+0.9, +0.9],
[+1.0, -1.0],
[+1.1, -0.9],
[+0.9, -1.1],
[-1.0, +1.0],
[-1.1, +0.9],
[-0.9, +1.1],
[-1.0, -1.0],
[-1.1, -1.1],
[-0.9, -0.9],
# NEXT STEP
[+1.0, +1.0],
[+1.1, +1.1],
# NEXT STEP
[+0.9, +0.9],
[+1.0, -1.0],
# NEXT STEP
[-1.2, -1.2],
[-0.8, -0.8]
])
in_stream.extend(points)
Stream.scheduler.step()
print
print 'num_dimensions = ', num_dimensions
print 'window_size = ', window_size
print 'step_size = ', step_size
print 'num_clusters = ', num_clusters
print 'points: '
print points
print 'output_stream: '
print recent_values(out_stream)
def test():
x = StreamArray(name='x', dtype=int)
y = StreamArray(name='y', dtype=int)
z = StreamArray(name='z', dtype=int)
multiply(x, y, window_size=10, multiplicand=4)
add(x, z, window_size=5, addend=100)
x.extend(np.arange(20))
Stream.scheduler.step()
print 'Testing multiply'
print 'multiplicand = 4'
print recent_values(y)
print
print 'Testing add'
print 'addend = 100'
print recent_values(z)
def test_iot_merge():
x = StreamArray(dtype=float)
y = StreamArray(dtype=float)
z = StreamArray(dimension=2, dtype=float)
def f(A_list, z):
"""
f is the function wrapped by an iot_merge agent.
A_list is a list of arrays. A_list[j] is the input array obtained
from the j-th input stream of the agent that wraps f.
z is the positional argument of f. z is an output stream that is
extended by f.
The agent completes reading n_rows elements of each array in
A_list where n_rows is the number of elements in the smallest
array. So, the function returns n_rows.
"""
n_rows = min([len(A) for A in A_list])
n_cols = len(A_list)
out = np.column_stack((A_list[0][:n_rows], A_list[1][:n_rows]))
z.extend(out)
return [n_rows for A in A_list]
# Create the agent by wrapping function f.
# A_list has two arrays from streams x and y.
# z is a keyword argument for f.
iot_merge(f, [x, y], z=z)
# Extend stream x with [0, 1, 2, 3, 4]
x.extend(np.arange(5, dtype=float))
run()
assert np.array_equal(recent_values(x), np.array(np.arange(5,
dtype=float)))
assert np.array_equal(recent_values(x), np.array([0., 1., 2., 3., 4.]))
assert np.array_equal(recent_values(y), np.zeros(shape=(0, ), dtype=float))
assert np.array_equal(recent_values(z), np.zeros(shape=(0, 2),
dtype=float))
y.extend(np.arange(100, 107, dtype=float))
run()
assert np.array_equal(recent_values(x), np.array([0., 1., 2., 3., 4.]))
assert np.array_equal(recent_values(y),
np.array([100., 101., 102., 103., 104., 105., 106.]))
assert np.array_equal(
recent_values(z),
np.array([[0., 100.], [1., 101.], [2., 102.], [3., 103.], [4., 104.]]))
def test_fmap_with_stream_array():
x = StreamArray(dimension=2, dtype=int)
@fmap_e
def g(v): return 2*v
y = g(x)
A = np.array([[1, 10], [2, 20], [3, 30]])
x.extend(A)
run()
assert np.array_equal(
recent_values(y),
[[2, 20], [4, 40], [6, 60]])
x.append(np.array([4, 40]))
run()
assert np.array_equal(
recent_values(y),
[[2, 20], [4, 40], [6, 60], [8, 80]])
def test_simple_array():
"""
Same as test_simple except that StreamArray is used in place
of Stream.
Create an agent with a single input stream array, x, and a single
output stream array, y. The elements of y are twice the corresponding
elements of x.
"""
# Create the streams arrays.
x = StreamArray(name='x', dtype=int)
y = StreamArray(name='y', dtype=int)
def f(array_of_int, s):
"""
Parameters
----------
f: func
the function that is wrapped by iot to create an
agent.
array_of_int: NumPy array of int
s: StreamArray
the output stream array of the agent.
Returns
-------
The function returns len(array_of_int) because the
agent has finished processing the entire array.
"""
s.extend(array_of_int * 2)
return len(array_of_int)
# Create the agent by wrapping function f.
iot(f, x, y)
# Extend input stream x of the agent with an array
x.extend(np.arange(5, dtype=int))
run()
assert np.array_equal(
recent_values(y), np.array([0, 2, 4, 6, 8]))
def test_sliding_window_with_startup():
x = StreamArray(dtype=int)
y = StreamArray(dtype=int)
sw = sliding_window_with_startup(func=np.sum,
in_stream=x,
out_stream=y,
window_size=10,
step_size=3)
# tests
x.extend(np.arange(6, dtype=int))
run()
assert np.array_equal(recent_values(y), np.array([3, 15]))
x.extend(np.arange(6, 9, dtype=int))
run()
assert np.array_equal(recent_values(y), np.array([3, 15, 36]))
x.extend(np.arange(9, 15, dtype=int))
run()
assert np.array_equal(recent_values(y), np.array([3, 15, 36, 65, 95]))
x.extend(np.arange(15, 30, dtype=int))
run()
assert np.array_equal(
recent_values(y),
np.array([3, 15, 36, 65, 95, 125, 155, 185, 215, 245]))
def test_plus_operator_with_arrays_1():
x = StreamArray(dtype=int)
y = StreamArray(dtype=int)
z = x + y
x.extend(np.arange(3))
y.extend(np.arange(100, 105))
run()
assert isinstance(recent_values(z), np.ndarray)
assert np.array_equal(recent_values(z), np.array([100, 102, 104]))
x.extend(np.arange(3, 7))
run()
assert np.array_equal(recent_values(z), np.array([
100, 102, 104, 106, 108]))
run()
assert np.array_equal(recent_values(z), np.array([
100, 102, 104, 106, 108]))
def test_plus_operator_with_arrays():
x = StreamArray(dimension=2, dtype=int)
y = StreamArray(dimension=2, dtype=int)
z = x + y
A = np.arange(6).reshape((3, 2))
B = np.arange(100, 110).reshape((5, 2))
x.extend(A)
y.extend(B)
run()
assert isinstance(z, StreamArray)
assert np.array_equal(recent_values(z), np.array([
[100, 102], [104, 106], [108, 110]]))
C = np.arange(6, 12).reshape((3, 2))
x.extend(C)
run()
assert np.array_equal(recent_values(z), np.array([
[100, 102], [104, 106], [108, 110],
[112, 114], [116, 118]]))
def test_some_merge_agents():
import numpy as np
scheduler = Stream.scheduler
#----------------------------------------------------
# Declare streams
s = Stream('s')
t = Stream('t')
u = Stream('u')
v_stream = Stream('v')
x = Stream('x')
#----------------------------------------------------
# Define functions
def g(lst):
return sum(lst)
def g_args(lst, multiplier):
return sum(lst) * multiplier
def general_f(lst, f):
return f(lst)
def fff(lst, f, addend):
return f(lst, addend)
def hhh(lst, addend):
return sum(lst) + addend
#----------------------------------------------------
# Define agents
d = zip_map(func=sum, in_streams=[x, u], out_stream=s, name='d')
def magnitude(vector):
return math.sqrt(sum([w * w for w in vector]))
ssssss = Stream()
ddd = zip_map(func=magnitude, in_streams=[x, u], out_stream=ssssss)
zipxu = zip_stream_f([x, u])
zip_map_xu = zip_map_f(sum, [x, u])
zip_map_xu_merge = Stream('zip map xu merge')
zip_map(sum, [x, u], zip_map_xu_merge)
zip_map_g_args = zip_map_f(g_args, [x, u], multiplier=2)
dd = zip_map(func=general_f,
in_streams=[x, u],
out_stream=t,
name='dd',
f=np.mean)
zip_map_ss = zip_map_f(np.mean, [x, u])
dddd = zip_map(func=fff,
in_streams=[x, u],
out_stream=v_stream,
name='dddd',
f=hhh,
addend=10)
#----------------------------------------------------
#----------------------------------------------------
# Append values to stream
x.extend(list(range(3)))
u.extend([10, 15, 18])
scheduler.step()
assert recent_values(s) == [10, 16, 20]
assert recent_values(zip_map_g_args) == [2 * v for v in recent_values(s)]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18)]
assert recent_values(t) == [5, 8, 10]
assert recent_values(zip_map_ss) == [5.0, 8.0, 10.0]
assert recent_values(v_stream) == [20, 26, 30]
assert recent_values(zip_map_xu) == s.recent[:s.stop]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
#----------------------------------------------------
u.append(37)
x.extend(list(range(3, 5, 1)))
scheduler.step()
assert recent_values(s) == [10, 16, 20, 40]
assert recent_values(zip_map_g_args) == [2 * v for v in recent_values(s)]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18), (3, 37)]
assert recent_values(t) == [5, 8, 10, 20]
assert recent_values(v_stream) == [20, 26, 30, 50]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
assert recent_values(ssssss) == [
10.0, 15.033296378372908, 18.110770276274835, 37.12142238654117
]
#----------------------------------------------------
u.extend([96, 95])
scheduler.step()
assert recent_values(s) == [10, 16, 20, 40, 100]
assert recent_values(zipxu) == [(0, 10), (1, 15), (2, 18), (3, 37),
(4, 96)]
assert recent_values(t) == [5, 8, 10, 20, 50]
assert recent_values(v_stream) == [20, 26, 30, 50, 110]
assert recent_values(zip_map_xu) == recent_values(zip_map_xu_merge)
#----------------------------------------------------
# TEST MERGE_ASYNCH AND MIX
#----------------------------------------------------
x = Stream('x')
y = Stream('y')
z = Stream('z')
w = Stream('w')
def g_asynch(pair):
index, value = pair
if index == 0:
return value * 10
elif index == 1:
return value * 2
else:
raise Exception()
merge_asynch(func=lambda v: v, in_streams=[x, y], out_stream=z)
merge_asynch(func=g_asynch, in_streams=[x, y], out_stream=w)
mix_z = mix_f([x, y])
scheduler.step()
assert recent_values(z) == []
assert recent_values(mix_z) == []
assert recent_values(w) == []
x.append(10)
scheduler.step()
assert recent_values(z) == [(0, 10)]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100]
y.append('A')
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A')]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100, 'AA']
y.append('B')
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A'), (1, 'B')]
assert recent_values(mix_z) == recent_values(z)
assert recent_values(w) == [100, 'AA', 'BB']
x.append(20)
scheduler.step()
assert recent_values(z) == [(0, 10), (1, 'A'), (1, 'B'), (0, 20)]
assert recent_values(z) == recent_values(mix_z)
assert recent_values(w) == [100, 'AA', 'BB', 200]
fahrenheit = Stream('fahrenheit')
celsius = Stream('celsius')
def fahrenheit_and_celsius(pair):
index, value = pair
if index == 0:
return (value - 32.0) / 1.8
elif index == 1:
return value
else:
raise Exception()
fahrenheit_stream = Stream('fahrenheit temperatures')
celsius_stream = Stream('celsius temperatures')
centigrade_stream = Stream('centigrade temperatures')
merge_asynch(func=fahrenheit_and_celsius,
in_streams=[fahrenheit_stream, celsius_stream],
out_stream=centigrade_stream)
fahrenheit_stream.append(32)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0]
fahrenheit_stream.append(50)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0]
fahrenheit_stream.append(68)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0, 20.0]
celsius_stream.append(-10.0)
scheduler.step()
assert recent_values(centigrade_stream) == [0.0, 10.0, 20.0, -10.0]
#----------------------------------------------------
# TEST BLEND
#----------------------------------------------------
x = Stream('x')
y = Stream('y')
z = Stream('z')
z_addend = Stream('z_addend')
def double(v):
return 2 * v
def double_add(v, addend):
return 2 * v + addend
blend(func=double, in_streams=[x, y], out_stream=z)
blend(func=double, in_streams=[x, y], out_stream=z_addend)
blend_z = blend_f(double, [x, y])
blend_add_z = blend_f(double_add, [x, y], addend=10)
x.append(1)
scheduler.step()
assert recent_values(z) == [2]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
x.extend(list(range(2, 4)))
scheduler.step()
assert recent_values(z) == [2, 4, 6]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
y.extend(list(range(100, 102)))
scheduler.step()
assert recent_values(z) == [2, 4, 6, 200, 202]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
x.extend([10, 20])
scheduler.step()
assert recent_values(z) == [2, 4, 6, 200, 202, 20, 40]
assert recent_values(blend_z) == recent_values(z)
assert recent_values(blend_add_z) == [v + 10 for v in recent_values(z)]
#----------------------------------------------------
# TEST MANY
#----------------------------------------------------
# func operates on a list with one element for each input stream.
# func returns a list with one element for each output stream.
def f_many(lst):
return [sum(lst), sum(lst) + 1]
u_stream = Stream(name='u_stream')
v_stream = Stream(name='v_stream')
w_stream = Stream(name='w_stream')
x_stream = Stream(name='x_stream')
multi_agent = multi_element(func=f_many,
in_streams=[u_stream, v_stream],
out_streams=[w_stream, x_stream],
name='multi_agent')
ww_stream, xx_stream = multi_element_f(func=f_many,
in_streams=[u_stream, v_stream],
num_out_streams=2)
u_stream.extend(list(range(5)))
v_stream.extend(list(range(0, 40, 4)))
scheduler.step()
assert recent_values(w_stream) == [0, 5, 10, 15, 20]
assert recent_values(x_stream) == [1, 6, 11, 16, 21]
assert recent_values(ww_stream) == recent_values(w_stream)
assert recent_values(xx_stream) == recent_values(x_stream)
# ------------------------------------
# Test many with args and kwargs
# func operates on a list with one element for each input stream.
# func returns a list with one element for each output stream.
def f_multi_args_kwargs(lst, multiplicand, addend):
return sum(lst) * multiplicand, sum(lst) + addend
u_args_kwargs_stream = Stream(name='u_args_kwargs_stream')
v_args_kwargs_stream = Stream(name='v_args_kwargs_stream')
w_args_kwargs_stream = Stream(name='w_args_kwargs_stream')
x_args_kwargs_stream = Stream(name='x_args_kwargs_stream')
multi_args_kwargs_agent = multi_element(
func=f_multi_args_kwargs,
in_streams=[u_args_kwargs_stream, v_args_kwargs_stream],
out_streams=[w_args_kwargs_stream, x_args_kwargs_stream],
name='multi_args_kwargs_agent',
multiplicand=2,
addend=10)
ww_args_kwargs_stream, xx_args_kwargs_stream = multi_element_f(
func=f_multi_args_kwargs,
in_streams=[u_args_kwargs_stream, v_args_kwargs_stream],
num_out_streams=2,
multiplicand=2,
addend=10)
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.extend(list(range(5)))
v_args_kwargs_stream.extend(list(range(0, 40, 4)))
scheduler.step()
assert recent_values(w_args_kwargs_stream) == [0, 10, 20, 30, 40]
assert recent_values(x_args_kwargs_stream) == [10, 15, 20, 25, 30]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.append(100)
v_args_kwargs_stream.extend(list(range(40, 80, 4)))
scheduler.step()
assert recent_values(w_args_kwargs_stream) == \
[0, 10, 20, 30, 40, 240]
assert recent_values(x_args_kwargs_stream) == \
[10, 15, 20, 25, 30, 130]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
u_args_kwargs_stream.extend([200, 300])
scheduler.step()
v_args_kwargs_stream.append(100)
scheduler.step()
assert recent_values(w_args_kwargs_stream) == \
[0, 10, 20, 30, 40, 240, 448, 656]
assert recent_values(x_args_kwargs_stream) == \
[10, 15, 20, 25, 30, 130, 234, 338]
assert (recent_values(ww_args_kwargs_stream) == recent_values(
w_args_kwargs_stream))
assert (recent_values(xx_args_kwargs_stream) == recent_values(
x_args_kwargs_stream))
#----------------------------------------------------
#----------------------------------------------------
# TEST STREAM ARRAY
#----------------------------------------------------
#----------------------------------------------------
#----------------------------------------------------
# Test zip_map with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
z = StreamArray('z')
a = StreamArray('a')
def sum_array_axis_0(a_list_of_arrays):
return np.sum(a_list_of_arrays, axis=0)
merge_list(func=sum_array_axis_0, in_streams=[x, y], out_stream=z)
def mean_array_axis_0(a_list_of_arrays):
return np.mean(a_list_of_arrays, axis=0)
zip_map_list(func=mean_array_axis_0, in_streams=[x, y], out_stream=a)
x.extend(np.linspace(0.0, 9.0, 10))
scheduler.step()
y.extend(np.linspace(0.0, 4.0, 5))
scheduler.step()
expected_array = np.sum(
[np.linspace(0.0, 4.0, 5),
np.linspace(0.0, 4.0, 5)], axis=0)
assert isinstance(z, StreamArray)
assert np.array_equal(recent_values(z), expected_array)
expected_means = np.linspace(0.0, 4.0, 5)
assert np.array_equal(recent_values(a), expected_means)
#----------------------------------------------------
# Test blend with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
z = StreamArray('z')
a = StreamArray('a')
def double(v):
return 2 * v
def double_add(v, addend):
return 2 * v + addend
## blend(func=double, in_streams=[x, y], out_stream=z)
## blend(func=double_add, in_streams=[x, y], out_stream=a, addend=10.0)
## x.append(np.array(1.0))
## scheduler.step()
## assert np.array_equal(recent_values(z), np.array([2.0]))
## assert np.array_equal(recent_values(a), recent_values(z)+10.0)
## x.extend(np.linspace(2.0, 3.0, 2))
## scheduler.step()
## assert np.array_equal(recent_values(z), np.array([2., 4., 6.]))
## assert np.array_equal(recent_values(a), recent_values(z)+10.0)
## y.extend(np.linspace(100.0, 101.0, 2))
## scheduler.step()
## assert np.array_equal(recent_values(z), [2., 4., 6., 200., 202.])
## assert np.array_equal(recent_values(a), recent_values(z)+10.0)
## x.extend([10., 20.])
## scheduler.step()
## assert np.array_equal(recent_values(z), [2., 4., 6., 200., 202., 20., 40.])
## assert np.array_equal(recent_values(a), recent_values(z)+10.0)
#----------------------------------------------------
# Test merge_asynch with StreamArray
#----------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
dt_0 = np.dtype([('time', int), ('value', float)])
z = StreamArray('z', dimension=2)
merge_asynch(func=lambda v: v, in_streams=[x, y], out_stream=z)
scheduler.step()
assert np.array_equal(recent_values(z), np.empty(shape=(0, 2)))
x.append(np.array(10.0))
scheduler.step()
assert np.array_equal(recent_values(z), np.array([(0, 10.0)]))
y.append(np.array(1.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.0)])
y.append(np.array(2.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.0), (1, 2.0)])
x.append(np.array(20.0))
scheduler.step()
assert np.array_equal(recent_values(z), [(0, 10.), (1, 1.), (1, 2.),
(0, 20.)])
#----------------------------------------------------------------
# Test window merge agent with no state
r = Stream('r')
w = Stream('w')
x = Stream('x')
a = Stream('a')
def h(list_of_windows):
return sum([sum(window) for window in list_of_windows])
merge_window(func=h,
in_streams=[r, w],
out_stream=x,
window_size=3,
step_size=3)
merge_stream = merge_window_f(func=h,
in_streams=[r, w],
window_size=3,
step_size=3)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window merge agent with state
def h_with_state(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_with_state,
in_streams=[r, w],
out_stream=a,
window_size=3,
step_size=3,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
r.extend(list(range(16)))
scheduler.step()
assert recent_values(r) == list(range(16))
assert recent_values(x) == []
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(a) == []
w.extend([10, 12, 14, 16, 18])
scheduler.step()
assert recent_values(r) == list(range(16))
assert recent_values(w) == [10, 12, 14, 16, 18]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(a) == [39]
#----------------------------------------------------------------
r.extend([10, -10, 21, -20])
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(a) == [39]
#----------------------------------------------------------------
w.append(20)
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(a) == [39, 67]
#----------------------------------------------------------------
r.extend([-1, 1, 0])
scheduler.step()
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(a) == [39, 67]
#----------------------------------------------------------------
# TEST MERGE_WINDOW WITH STREAM ARRAY
#----------------------------------------------------------------
x = StreamArray('x', dimension=2)
b = StreamArray('b', dimension=2)
a = StreamArray('a', dimension=2)
#----------------------------------------------------------------
# Test window merge agent with state
def h_array(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_array,
in_streams=[x, a],
out_stream=b,
window_size=2,
step_size=2,
state=0)
#----------------------------------------------------------------
x.extend(np.array([[1., 5.], [7., 11.]]))
a.extend(np.array([[0., 1.], [2., 3.]]))
scheduler.step()
np.array_equal(recent_values(b), np.empty(shape=(0, 2)))
a.extend(np.array([[0., 1.], [1., 0.]]))
scheduler.step()
np.array_equal(recent_values(b), np.empty(shape=(0, 2)))
x.extend(np.array([[14., 18.], [18., 30.], [30., 38.], [34., 42.]]))
scheduler.step()
#-------------------------------------------------------------------
# TEST MERGE_LIST
#-------------------------------------------------------------------
# Function g operates on a list of lists, one list for each input
# stream, to return a single list for the output stream.
x = Stream('x list merge')
u = Stream('u list merge')
s = Stream('s list merge')
def g(list_of_lists):
return [sum(snapshot) for snapshot in list(zip(*list_of_lists))]
d = merge_list(func=g, in_streams=[x, u], out_stream=s, name='d')
ss = merge_list_f(g, [x, u])
x.extend(list(range(4)))
u.extend(list(range(10, 20, 2)))
scheduler.step()
assert recent_values(x) == [0, 1, 2, 3]
assert recent_values(u) == [10, 12, 14, 16, 18]
assert recent_values(s) == [10, 13, 16, 19]
x = StreamArray()
y = StreamArray()
z = StreamArray(dtype='bool')
def f(two_lists):
return np.array(two_lists[0]) > np.array(two_lists[1])
merge_list(f, [x, y], z)
x.extend(np.array([3.0, 5.0, 7.0]))
y.extend(np.array([4.0, 3.0, 10.0]))
run()
def test_sink():
import numpy as np
scheduler = Stream.scheduler
## ----------------------------------------------
## # Examples from AssembleSoftware website: KEEP!!
## ----------------------------------------------
## def print_index(v, state, delimiter):
## print str(state) + delimiter + str(v)
## return state+1 # next state
## s = Stream()
## sink(print_index, s, 0, delimiter=':')
## s.extend(range(100,105))
## s = Stream()
## def print_index(v, state, delimiter):
## print str(state) + delimiter + str(v)
## return state+1 # next state
## sink(print_index, s, 0, delimiter=':')
## s.extend(range(100,105))
# Set up parameters for call to stream_to_list
## ----------------------------------------------
## # Finished examples from AssembleSoftware website
## ----------------------------------------------
#-----------------------------------------------
# Set up parameters for call to sink
print_list = []
print_list_for_array = []
def print_index(v, state, print_list):
print_list.append(str(state) + ':' + str(v))
return state + 1 # next state
s = Stream('s')
s_array = StreamArray('s_array', dtype=int)
#-----------------------------------------------
# Call sink with initial state of 0
sink(func=print_index, in_stream=s, state=0, print_list=print_list)
sink(func=print_index,
in_stream=s_array,
state=0,
print_list=print_list_for_array)
s.extend(range(100, 103))
s_array.extend(np.arange(100, 103))
scheduler.step()
assert print_list == ['0:100', '1:101', '2:102']
assert print_list_for_array == print_list
s.extend(range(200, 203))
scheduler.step()
assert print_list == ['0:100', '1:101', '2:102', '3:200', '4:201', '5:202']
#-----------------------------------------------
input_stream = Stream('input stream')
input_stream_array = StreamArray('input stream array', dtype=int)
output_list = []
output_list_array = []
# Call stream_to_list with no function
stream_to_list(input_stream, output_list)
stream_to_list(input_stream_array, output_list_array)
# A test
a_test_list = range(100, 105)
a_test_array = np.arange(100, 105)
input_stream.extend(a_test_list)
input_stream_array.extend(a_test_array)
scheduler.step()
assert output_list == a_test_list
assert output_list_array == a_test_list
#-----------------------------------------------
# test stream to list with a function
def h(v, multiplier, addend):
return v * multiplier + addend
ss = Stream('ss')
ss_array = StreamArray('ss_array', dtype=int)
l = []
l_array = []
stream_to_list(ss, l, h, multiplier=2, addend=100)
stream_to_list(in_stream=ss_array,
target_list=l_array,
element_function=h,
multiplier=2,
addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
ss_array.extend(np.array(test_list))
scheduler.step()
assert l == [v * 2 + 100 for v in test_list]
assert l_array == l
#-----------------------------------------------
# test stream to list with a function and state
def h(v, state, multiplier, addend):
return v * multiplier + addend + state, v + state
ss = Stream('ss')
ss_array = StreamArray('ss_array', dtype=int)
l = []
l_array = []
stream_to_list(ss, l, h, 0, multiplier=2, addend=100)
stream_to_list(in_stream=ss_array,
target_list=l_array,
element_function=h,
state=0,
multiplier=2,
addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
ss_array.extend(np.array(test_list))
scheduler.step()
assert l == [106, 149, 154]
assert l_array == l
ss = Stream('ss')
ss_array = StreamArray('ss_array', dtype=int)
l = []
l_array = []
stream_to_list(ss, l, h, 0, multiplier=2, addend=100)
stream_to_list(in_stream=ss_array,
target_list=l_array,
element_function=h,
state=0,
multiplier=2,
addend=100)
test_list = range(5)
ss.extend(test_list)
ss_array.extend(np.array(test_list))
scheduler.step()
assert l == [100, 102, 105, 109, 114]
assert l_array == l
# Test sink
# func operates on a single element of the single input stream and does
# not return any value.
def p(v, lst):
lst.append(v)
in_stream_sink = Stream('in_stream_sink')
a_list = []
b_list = []
sink_agent = sink_element(func=p,
in_stream=in_stream_sink,
name='sink_agent',
lst=a_list)
sink(func=p, in_stream=in_stream_sink, lst=b_list)
test_list = [1, 13, 29]
in_stream_sink.extend(test_list)
scheduler.step()
assert a_list == test_list
assert b_list == test_list
# ------------------------------------
# Test sink with state
# func operates on a single element of the single input stream and state.
# func does not return any value.
def p_s(element, state, lst, stream_name):
lst.append([stream_name, element])
return state + 1
in_stream_sink_with_state = Stream('s')
c_list = []
sink_with_state_agent = sink_element(func=p_s,
in_stream=in_stream_sink_with_state,
state=0,
name='sink_with_state_agent',
lst=c_list,
stream_name='s')
#------------------------------------------------------------------------------
# Test sink as a function with state
d_list = []
sink(p_s, in_stream_sink_with_state, state=0, lst=d_list, stream_name='s')
in_stream_sink_with_state.extend(range(2))
scheduler.step()
assert c_list == [['s', 0], ['s', 1]]
assert d_list == c_list
# ------------------------------------
# Test sink with side effect
# func operates on a single element of the single input stream and state.
# func does not return any value.
def sink_with_side_effect_func(element, side_effect_list, f):
side_effect_list.append(f(element))
return None
side_effect_list_0 = []
side_effect_list_1 = []
side_effect_list_2 = []
def ff(element):
return element * 2
def fff(element):
return element + 10
stm = Stream('stm')
sink_with_side_effect_agent_0 = sink_element(
func=sink_with_side_effect_func,
in_stream=stm,
name='sink_with_side_effect_agent_0',
side_effect_list=side_effect_list_0,
f=ff)
sink_with_side_effect_agent_1 = sink_element(
func=sink_with_side_effect_func,
in_stream=stm,
name='sink_with_side_effect_agent_1',
side_effect_list=side_effect_list_1,
f=fff)
def f_stateful(element, state):
return element + state, element + state
def f_stateful_2(element, state):
return element * state, element + state
target_stream_to_list_simple = []
stream_to_list(stm, target_stream_to_list_simple)
stream_to_list(in_stream=stm,
target_list=side_effect_list_2,
element_function=lambda v: 2 * v)
target_stream_to_list_stateful = []
stream_to_list(in_stream=stm,
target_list=target_stream_to_list_stateful,
element_function=f_stateful,
state=0)
target_stream_to_list_stateful_2 = []
stream_to_list(in_stream=stm,
target_list=target_stream_to_list_stateful_2,
element_function=f_stateful_2,
state=0)
stream_to_file(stm, 'test1.txt')
stream_to_file(stm, 'test2.txt', lambda v: 2 * v)
stream_to_file(stm, 'test3.txt', f_stateful, state=0)
import Queue
queue_1 = Queue.Queue()
queue_2 = Queue.Queue()
queue_3 = Queue.Queue()
stream_to_queue(stm, queue_1)
stream_to_queue(stm, queue_2, lambda v: 2 * v)
stream_to_queue(stm, queue_3, f_stateful, 0)
stm.extend(range(5))
scheduler.step()
assert target_stream_to_list_stateful == [0, 1, 3, 6, 10]
assert target_stream_to_list_stateful_2 == [0, 0, 2, 9, 24]
assert side_effect_list_0 == [0, 2, 4, 6, 8]
assert side_effect_list_1 == [10, 11, 12, 13, 14]
assert side_effect_list_0 == side_effect_list_2
assert target_stream_to_list_simple == range(5)
with open('test1.txt') as the_file:
file_contents_integers = [int(v) for v in (the_file.readlines())]
assert file_contents_integers == recent_values(stm)
with open('test2.txt') as the_file:
file_contents_integers = [int(v) for v in (the_file.readlines())]
assert file_contents_integers == [2 * v for v in recent_values(stm)]
with open('test3.txt') as the_file:
file_contents_integers = [int(v) for v in (the_file.readlines())]
assert file_contents_integers == [0, 1, 3, 6, 10]
os.remove('test1.txt')
os.remove('test2.txt')
os.remove('test3.txt')
def h(v, multiplier, addend):
return v * multiplier + addend
ss = Stream()
stream_to_file(ss, 'test4.txt', h, multiplier=2, addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
scheduler.step()
with open('test4.txt') as the_file:
file_contents_integers = [int(v) for v in (the_file.readlines())]
assert file_contents_integers == [v * 2 + 100 for v in test_list]
os.remove('test4.txt')
def h(v, state, multiplier, addend):
return v * multiplier + addend + state, v + state
ss = Stream()
stream_to_file(ss, 'test5.txt', h, 0, multiplier=2, addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
scheduler.step()
with open('test5.txt') as the_file:
file_contents_integers = [int(v) for v in (the_file.readlines())]
scheduler.step()
assert file_contents_integers == [106, 149, 154]
os.remove('test5.txt')
# ------------------------------------
# Testing stream_to_queue
def h(v, state, multiplier, addend):
return v * multiplier + addend + state, v + state
ss = Stream()
queue_4 = Queue.Queue()
stream_to_queue(ss, queue_4, h, 0, multiplier=2, addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
scheduler.step()
queue_contents = []
while not queue_4.empty():
queue_contents.append(queue_4.get())
assert queue_contents == [106, 149, 154]
# Test with state and keyword arguments
def h(v, state, multiplier, addend):
return v * multiplier + addend + state, v + state
ss = Stream()
stream_to_queue(ss, queue_4, h, 0, multiplier=2, addend=100)
test_list = [3, 23, 14]
ss.extend(test_list)
queue_contents = []
scheduler.step()
while not queue_4.empty():
queue_contents.append(queue_4.get())
assert queue_contents == [106, 149, 154]
# Another test with state and keyword arguments
ss = Stream()
queue_5 = Queue.Queue()
stream_to_queue(ss, queue_5, h, 0, multiplier=2, addend=100)
test_list = range(5)
ss.extend(test_list)
scheduler.step()
queue_contents = []
while not queue_5.empty():
queue_contents.append(queue_5.get())
assert queue_contents == [100, 102, 105, 109, 114]
# Test stream_to_buffer
s = Stream()
buf = Buffer(max_size=10)
stream_to_buffer(s, buf)
test_list = range(5)
s.extend(test_list)
scheduler.step()
assert buf.get_all() == test_list
next_test = range(5, 10, 1)
s.extend(next_test)
scheduler.step()
assert buf.read_all() == next_test
assert buf.get_all() == next_test
s = Stream('s')
print_list = []
def f(lst):
print_list.extend(lst)
sink_window(func=f, in_stream=s, window_size=4, step_size=2)
s.extend(range(10))
scheduler.step()
assert print_list == [0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9]
print 'TEST OF SINK IS SUCCESSFUL'
return
def stream_test():
# Numpy type for testing stream array.
txyz_dtype = np.dtype([('time', 'int'), ('data', '3float')])
#--------------------------------------------
# Testing StreamArray with positive dimension
s = StreamArray(name='s', dimension=3)
# Each element of s is a numpy array with with 3 elements
# Initially s is empty. So s.stop == 0
assert s.stop == 0
# If num_in_memory is not specified in the declaration for
# s, the default value, DEFAULT_NUM_IN_MEMORY, is used.
# The length of s.recent is twice num_in_memory
assert len(s.recent) == 2 * DEFAULT_NUM_IN_MEMORY
# Append a numpy array with 3 zeros to s
s.append(np.zeros(3))
assert (s.stop == 1)
# Thus s.recent[:s.stop] is an array with 1 row and 3 columns.
assert (np.array_equal(s.recent[:s.stop], np.array([[0.0, 0.0, 0.0]])))
# Extend s by an array with 2 rows and 3 columns. The number of
# columns must equal the dimension of the stream array.
s.extend(np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]))
# s.stop is incremented to account for the addition of two elements.
assert s.stop == 3
# s.recent[:s.stop] includes all the elements added to s.
# Thus s.recent[:s.stop] is an array with 3 rows and 3 columns.
assert (np.array_equal(
s.recent[:s.stop],
np.array([[0.0, 0.0, 0.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])))
# Extend s by an array with 1 row and 3 columns. The number of
# columns must equal the dimension of the stream array.
s.extend(np.array([[7.0, 8.0, 9.0]]))
# s.stop is incremented to account for the addition of a single row.
assert s.stop == 4
# Thus s.recent[:s.stop] is an array with 4 rows and 3 columns.
assert np.array_equal(
s.recent[:s.stop],
np.array([[0.0, 0.0, 0.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0],
[7.0, 8.0, 9.0]]))
# Note the difference between EXTENDING s with an array consisting
# of 1 row and 3 columns, versus APPENDING a rank-1 array consisting
# of 3 elements, as in the following example.
s.append(np.array([10.0, 11.0, 12.0]))
# s.stop is incremented to account for the addition of a single row.
assert s.stop == 5
# Thus s.recent[:s.stop] is an array with 5 rows and 3 columns.
assert np.array_equal(
s.recent[:s.stop],
np.array([[0.0, 0.0, 0.0], [1.0, 2.0, 3.0], [4.0, 5.0, 6.0],
[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]))
#---------------------------------------------------------------
# Testing StreamArray with zero dimension and user-defined dtype
t = StreamArray(name='t', dimension=0, dtype=txyz_dtype)
# Each element of t is an object of dtype txyz_dtype
# t[i]['time'] is an int.
# t[i]['data'] is a 3-tuple consisting of 3 floats.
# Nothing has been appended to t, and so t.stop == 0.
assert (t.stop == 0)
# Append an object with 'time' = 1, and 'data' = [0.0, 1.0, 2.0].
t.append(np.array((1, [0.0, 1.0, 2.0]), dtype=txyz_dtype))
# Increase t.stop to account for the element that has just been
# added to t.
assert t.stop == 1
# t.recent[:t.stop] contains all elements appended to t.
assert t.recent[:t.stop] == np.array([(1, [0.0, 1.0, 2.0])],
dtype=txyz_dtype)
assert t.recent[0]['time'] == np.array(1)
assert np.array_equal(t.recent[0]['data'], np.array([0., 1., 2.]))
# Append another element to t.
t.append(np.array((2, [11.0, 12.0, 13.0]), dtype=txyz_dtype))
# Increase t.stop to account for the element that has just been
# added to t.
assert (t.stop == 2)
# t.recent[:t.stop] contains all elements appended to t.
a = np.array([(1, [0.0, 1.0, 2.0]), (2, [11.0, 12.0, 13.0])],
dtype=txyz_dtype)
assert np.array_equal(t.recent[:t.stop], a)
# Extend t by a list of 2 elements each of which consists of
# zeroes of txyz_dtype
t.extend(np.zeros(2, dtype=txyz_dtype))
# Increase t.stop to account for the element that has just been
# added to t.
assert (t.stop == 4)
# t.recent[:t.stop] contains all elements appended to t.
a = np.array([(1, [0.0, 1.0, 2.0]), (2, [11.0, 12.0, 13.0]),
(0, [0.0, 0.0, 0.0]), (0, [0.0, 0.0, 0.0])],
dtype=txyz_dtype)
assert np.array_equal(t.recent[:t.stop], a)
#---------------------------------------------------------------
# Testing simple Stream
u = Stream('u')
v = Stream('v')
# Add elements 0, 1, 2, 3 to stream u.
u.extend(list(range(4)))
# Increase u.stop to account for the element that has just been
# added to u.
assert u.stop == 4
# u.recent[:t.stop] contains all elements appended to u.
assert u.recent[:u.stop] == [0, 1, 2, 3]
# No change to v.
assert v.stop == 0
# Append element 10 to v and then append the list [40, 50]
v.append(10)
v.append([40, 50])
# Increase v.stop by 2 to account for the 2 new elements appended
# to v.
assert v.stop == 2
# v.recent[:v.stop] contains all elements appended to v.
assert v.recent[:v.stop] == [10, [40, 50]]
# Extend stream v
v.extend([60, 70, 80])
# Increase v.stop by 3 to account for the 3 new elements appended
# to v.
assert v.stop == 5
# v.recent[:v.stop] contains all elements appended to v.
assert v.recent[:v.stop] == [10, [40, 50], 60, 70, 80]
#------------------------------------------
# Test helper functions: get_contents_after_column_value()
# Also test StreamArray
y = StreamArray(name='y', dimension=0, dtype=txyz_dtype, num_in_memory=64)
# y[i]['time'] is a time (int).
# y[i]['data'] is 3-tuple usually with directional data
# for x, y, z.
# y.recent length is twice num_in_memory
assert len(y.recent) == 128
# y has no elements, so y.stop == 0
assert y.stop == 0
# Test data for StreamArray with user-defined data type.
test_data = np.zeros(128, dtype=txyz_dtype)
assert len(test_data) == 128
# Put random numbers for test_data[i]['time'] and
# test_data[i]['data'][xyx] for xyz in [0, 1, 2]
for i in range(len(test_data)):
test_data[i]['time'] = random.randint(0, 1000)
for j in range(3):
test_data[i]['data'][j] = random.randint(2000, 9999)
# ordered_test_data has time in increasing order.
ordered_test_data = np.copy(test_data)
for i in range(len(ordered_test_data)):
ordered_test_data[i]['time'] = i
y.extend(ordered_test_data[:60])
# extending y does not change length of y.recent
assert (len(y.recent) == 128)
# y.stop increases to accommodate the extension of y by 60.
assert (y.stop == 60)
# y.recent[:y.stop] now contains all the values put into y.
assert np.array_equal(y.recent[:y.stop], ordered_test_data[:60])
assert np.array_equal(
y.get_contents_after_column_value(column_number=0, value=50),
ordered_test_data[50:60])
assert np.array_equal(y.get_contents_after_time(start_time=50),
ordered_test_data[50:60])
assert (y.get_index_for_column_value(column_number=0, value=50) == 50)
yz = StreamArray(name='yz',
dimension=0,
dtype=txyz_dtype,
num_in_memory=64)
c = np.array((1, [0., 1., 2.]), dtype=txyz_dtype)
yz.append(c)
assert np.array_equal(yz.recent[:yz.stop],
np.array([(1, [0., 1., 2.])], dtype=txyz_dtype))
d = np.array([(2, [3., 4., 5.]), (3, [6., 7., 8.])], dtype=txyz_dtype)
yz.extend(d)
assert np.array_equal(
yz.recent[:yz.stop],
np.array([(1, [0., 1., 2.]), (2, [3., 4., 5.]), (3, [6., 7., 8.])],
dtype=txyz_dtype))
#------------------------------------------
# TESTING regular Stream class
x = Stream(name='x', num_in_memory=8)
# The length of x.recent is twice num_in_memory
assert (len(x.recent) == 16)
# No values have been appended to stream x; so x.stop == 0
assert (x.stop == 0)
# Test append
x.append(10)
# Appending values to x does not change len(x.recent)
assert (len(x.recent) == 16)
# x.stop increases to accomodate the value appended.
assert (x.stop == 1)
# x.recent[:x.stop] includes the latest append
assert (x.recent[:x.stop] == [10])
x.append(20)
assert (len(x.recent) == 16)
# x.stop increases to accomodate the value appended.
assert (x.stop == 2)
# x.recent[:x.stop] includes the latest append
assert (x.recent[:2] == [10, 20])
# Test extend
x.extend([30, 40])
assert (len(x.recent) == 16)
# x.stop increases to accomodate the values extended.
assert (x.stop == 4)
# x.recent[:x.stop] includes the latest extend
assert (x.recent[:x.stop] == [10, 20, 30, 40])
# Checking extension with the empty list.
x.extend([])
assert (len(x.recent) == 16)
# extending a stream with the empty list does not change
# the stream.
assert (x.stop == 4)
assert (x.recent[:4] == [10, 20, 30, 40])
# Checking extending a stream with a singleton list
x.extend([50])
assert (len(x.recent) == 16)
assert (x.stop == 5)
assert (x.recent[:5] == [10, 20, 30, 40, 50])
# Check registering a reader.
# Register a reader called 'a' for stream x starting
# to read from x[3] onwards.
x.register_reader('a', 3)
# Register a reader called 'b' for stream x starting
# to read from x[4] onwards.
x.register_reader('b', 4)
# x.start is a dict which identifies the readers of x
# and where they are starting to read from.
assert (x.start == {'a': 3, 'b': 4})
x.extend([1, 2, 3, 4, 5])
assert (len(x.recent) == 16)
assert (x.stop == 10)
assert (x.recent[:10] == [10, 20, 30, 40, 50, 1, 2, 3, 4, 5])
assert (x.start == {'a': 3, 'b': 4})
x.register_reader('a', 7)
x.register_reader('b', 7)
assert (x.start == {'a': 7, 'b': 7})
#------------------------------------------
# Test helper functions
assert (x.get_last_n(n=2) == [4, 5])
v = StreamArray(dimension=(3, 4), dtype=int)
v.append(np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]))
a = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]])
np.array_equal(v.recent[:v.stop],
np.array([[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]]]))
v.extend(
np.array([[[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]],
[[24, 25, 26, 27], [28, 29, 30, 31], [32, 33, 34, 35]]]))
np.array_equal(
v.recent[:v.stop],
np.array([[[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]],
[[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]],
[[24, 25, 26, 27], [28, 29, 30, 31], [32, 33, 34, 35]]]))
u = StreamArray(name='u', dimension=2, dtype=int)
a = np.array([0, 1])
u.append(a)
np.array_equal(u.recent[:u.stop], np.array([[0, 1]]))
u.extend(np.array([[2, 3], [4, 5], [6, 7]]))
np.array_equal(u.recent[:u.stop], np.array([[0, 1], [2, 3], [4, 5], [6,
7]]))
t = StreamArray('t')
t.append(np.array(1.0))
t.extend(np.array([2.0, 3.0]))
np.array_equal(t.recent[:t.stop], np.array([1.0, 2.0, 3.0]))
def test_list():
scheduler = Stream.scheduler
n = Stream('n')
o = Stream('o')
p = Stream('p')
q = Stream('q')
r = Stream('r')
s = Stream('s')
t = Stream('t')
u = Stream('u')
v = Stream('v')
w = Stream('w')
x = Stream('x')
y = Stream('y')
z = Stream('z')
#-------------------------------------------------------------------
# Test simple map
def simple(lst):
return [2 * v for v in lst]
a = map_list(func=simple, in_stream=x, out_stream=y, name='a')
yy = map_list_f(simple, x)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test map with state
# Function that operates on an element and state and returns an
# element and state.
def f(input_list, state):
output_list = [[]] * len(input_list)
for i in range(len(input_list)):
output_list[i] = input_list[i] + state
state += 2
return output_list, state
b = map_list(func=f, in_stream=x, out_stream=z, state=0, name='b')
zz = map_list_f(f, x, 0)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test map with call streams
c = map_list(func=f,
in_stream=x,
out_stream=v,
state=10,
call_streams=[w],
name='c')
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test sink with state
def sink_with_state(input_list, output_list):
# sink has no output stream.
# This function only returns the next state.
return output_list.extend(input_list)
out_list = []
# In this simple example, out_list will be the same as the input
# stream.
sink_agent = sink_list(func=sink_with_state,
in_stream=x,
name='sink_agent',
state=out_list)
out_list_stream = []
# Function version of the previous agent example
sink_list_f(sink_with_state, x, out_list_stream)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test merge
# Function that operates on a list of lists
def g(list_of_lists):
return [sum(snapshot) for snapshot in zip(*list_of_lists)]
d = merge_list(func=g, in_streams=[x, u], out_stream=s, name='d')
ss = merge_list_f(g, [x, u])
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test split
def h(input_list):
return [[element + 1 for element in input_list],
[element * 2 for element in input_list]]
e = split_list(func=h, in_stream=x, out_streams=[r, t], name='e')
rr, tt = split_list_f(h, x, num_out_streams=2)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test split with state
def h_state(input_list, state):
length = len(input_list)
output_list_0 = [[]] * length
output_list_1 = [[]] * length
for i in range(length):
output_list_0[i] = input_list[i] + state
output_list_1[i] = input_list[i] * state
state += 1
return ([output_list_0, output_list_1], state)
split_list(func=h_state, in_stream=x, out_streams=[p, q], state=0)
pp, qq = split_list_f(h_state, x, num_out_streams=2, state=0)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
# Test many
def f_many(list_of_lists):
snapshots = zip(*list_of_lists)
return [[max(snapshot) for snapshot in snapshots],
[min(snapshot) for snapshot in snapshots]]
multi_agent = multi_list(func=f_many,
in_streams=[x, u],
out_streams=[n, o],
name='multi_agent')
nn, oo = multi_list_f(func=f_many, in_streams=[x, u], num_out_streams=2)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
#-------------------------------------------------------------------
x.extend(range(5))
scheduler.step()
assert recent_values(x) == range(5)
assert recent_values(y) == [0, 2, 4, 6, 8]
assert recent_values(z) == [0, 3, 6, 9, 12]
assert recent_values(v) == []
assert out_list == range(5)
assert out_list == out_list_stream
assert recent_values(s) == []
assert recent_values(r) == [1, 2, 3, 4, 5]
assert recent_values(t) == [0, 2, 4, 6, 8]
assert recent_values(p) == [0, 2, 4, 6, 8]
assert recent_values(q) == [0, 1, 4, 9, 16]
assert recent_values(n) == []
assert recent_values(o) == []
assert recent_values(y) == recent_values(yy)
assert recent_values(z) == recent_values(zz)
assert recent_values(s) == recent_values(ss)
assert recent_values(r) == recent_values(rr)
assert recent_values(t) == recent_values(tt)
assert recent_values(p) == recent_values(pp)
assert recent_values(q) == recent_values(qq)
assert recent_values(n) == recent_values(nn)
assert recent_values(o) == recent_values(oo)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
w.append(0)
scheduler.step()
assert recent_values(x) == range(5)
assert recent_values(y) == [0, 2, 4, 6, 8]
assert recent_values(z) == [0, 3, 6, 9, 12]
assert recent_values(v) == [10, 13, 16, 19, 22]
assert out_list == range(5)
assert recent_values(s) == []
assert recent_values(r) == [1, 2, 3, 4, 5]
assert recent_values(t) == [0, 2, 4, 6, 8]
assert recent_values(p) == [0, 2, 4, 6, 8]
assert recent_values(q) == [0, 1, 4, 9, 16]
assert recent_values(n) == []
assert recent_values(o) == []
assert recent_values(y) == recent_values(yy)
assert recent_values(z) == recent_values(zz)
assert recent_values(s) == recent_values(ss)
assert recent_values(r) == recent_values(rr)
assert recent_values(t) == recent_values(tt)
assert recent_values(p) == recent_values(pp)
assert recent_values(q) == recent_values(qq)
assert recent_values(n) == recent_values(nn)
assert recent_values(o) == recent_values(oo)
#-------------------------------------------------------------------
#-------------------------------------------------------------------
u.extend([10, 15, 18])
scheduler.step()
assert recent_values(s) == [10, 16, 20]
assert recent_values(n) == [10, 15, 18]
assert recent_values(o) == [0, 1, 2]
u.append(37)
scheduler.step()
assert recent_values(s) == [10, 16, 20, 40]
assert recent_values(n) == [10, 15, 18, 37]
assert recent_values(o) == [0, 1, 2, 3]
u.extend([96, 95])
scheduler.step()
assert recent_values(x) == range(5)
assert recent_values(y) == [0, 2, 4, 6, 8]
assert recent_values(z) == [0, 3, 6, 9, 12]
assert recent_values(v) == [10, 13, 16, 19, 22]
assert out_list == range(5)
assert recent_values(s) == [10, 16, 20, 40, 100]
assert recent_values(r) == [1, 2, 3, 4, 5]
assert recent_values(t) == [0, 2, 4, 6, 8]
assert recent_values(p) == [0, 2, 4, 6, 8]
assert recent_values(q) == [0, 1, 4, 9, 16]
assert recent_values(n) == [10, 15, 18, 37, 96]
assert recent_values(o) == [0, 1, 2, 3, 4]
assert recent_values(y) == recent_values(yy)
assert recent_values(z) == recent_values(zz)
assert recent_values(s) == recent_values(ss)
assert recent_values(r) == recent_values(rr)
assert recent_values(t) == recent_values(tt)
assert recent_values(p) == recent_values(pp)
assert recent_values(q) == recent_values(qq)
assert recent_values(n) == recent_values(nn)
assert recent_values(o) == recent_values(oo)
#------------------------------------------------------------------
#------------------------------------------------------------------
# Test NumPy arrays: StreamArray
#------------------------------------------------------------------
#------------------------------------------------------------------
# Test list map on StreamArray (dimension is 0).
a_stream_array = StreamArray(name='a_stream_array')
b_stream_array = StreamArray(name='b_stream_array')
def f_np(input_array):
return input_array + 1
a_np_agent = map_list(func=f_np,
in_stream=a_stream_array,
out_stream=b_stream_array,
name='a_np_agent')
bb_stream_array = map_array_f(f_np, a_stream_array)
scheduler.step()
assert np.array_equal(recent_values(b_stream_array),
np.array([], dtype=np.float64))
assert np.array_equal(recent_values(b_stream_array),
recent_values(bb_stream_array))
a_stream_array.extend(np.arange(5.0))
scheduler.step()
assert np.array_equal(recent_values(b_stream_array), np.arange(5.0) + 1)
assert np.array_equal(recent_values(b_stream_array),
recent_values(bb_stream_array))
a_stream_array.extend(np.arange(5.0, 10.0, 1.0))
scheduler.step()
assert np.array_equal(recent_values(b_stream_array), np.arange(10.0) + 1)
assert np.array_equal(recent_values(b_stream_array),
recent_values(bb_stream_array))
# Test list map with state on StreamArray (dimension is 0)
c_stream_array = StreamArray(name='c_stream_array')
d_stream_array = StreamArray(name='d_stream_array')
def f_np_state(input_array, state):
return np.cumsum(input_array) + state, np.sum(input_array)
b_np_agent = map_list(func=f_np_state,
in_stream=c_stream_array,
out_stream=d_stream_array,
state=0.0,
name='b_np_agent')
dd_stream_array = map_array_f(f_np_state, c_stream_array, state=0.0)
scheduler.step()
assert np.array_equal(recent_values(d_stream_array),
np.array([], dtype=np.float64))
assert np.array_equal(recent_values(d_stream_array),
recent_values(dd_stream_array))
c_stream_array.extend(np.arange(5.0))
scheduler.step()
assert np.array_equal(d_stream_array.recent[:d_stream_array.stop],
np.cumsum(np.arange(5.0)))
assert np.array_equal(recent_values(d_stream_array),
recent_values(dd_stream_array))
c_stream_array.extend(np.arange(5.0, 10.0, 1.0))
scheduler.step()
assert np.array_equal(d_stream_array.recent[:d_stream_array.stop],
np.cumsum(np.arange(10.0)))
assert np.array_equal(recent_values(d_stream_array),
recent_values(dd_stream_array))
# Test list map with positive integer dimension on StreamArray
e_stream_array = StreamArray(name='e_stream_array', dimension=3)
f_stream_array = StreamArray(name='f_stream_array', dimension=2)
def f_np_dimension(input_array):
output_array = np.zeros([len(input_array), 2])
output_array[:, 0] = input_array[:, 0] + input_array[:, 1]
output_array[:, 1] = input_array[:, 2]
return output_array
c_np_agent = map_list(func=f_np_dimension,
in_stream=e_stream_array,
out_stream=f_stream_array,
name='c_np_agent')
e_stream_array.extend(np.array([[1.0, 2.0, 3.0]]))
scheduler.step()
assert np.array_equal(f_stream_array.recent[:f_stream_array.stop],
np.array([[3.0, 3.0]]))
e_stream_array.extend(np.array([[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]))
scheduler.step()
assert np.array_equal(f_stream_array.recent[:f_stream_array.stop],
np.array([[3.0, 3.0], [9.0, 6.0], [15.0, 9.0]]))
# Test list map with a dimension which is a tuple of integers.
g_stream_array = StreamArray(name='g_stream_array', dimension=(2, 2))
h_stream_array = StreamArray(name='h_stream_array', dimension=(2, 2))
def f_np_tuple_dimension(input_array):
return input_array * 2
d_np_agent = map_list(func=f_np_tuple_dimension,
in_stream=g_stream_array,
out_stream=h_stream_array,
name='d_np_agent')
a_array = np.array([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]])
g_stream_array.extend(a_array)
scheduler.step()
assert np.array_equal(h_stream_array.recent[:h_stream_array.stop],
a_array * 2)
b_array = np.array([[[9.0, 10.0], [11.0, 12.0]]])
g_stream_array.extend(b_array)
scheduler.step()
assert np.array_equal(h_stream_array.recent[:h_stream_array.stop],
np.vstack((a_array, b_array)) * 2)
# Test list map with a datatype and dimension of 0.
dt_0 = np.dtype([('time', int), ('value', (float, 3))])
dt_1 = np.dtype([('time', int), ('value', float)])
i_stream_array = StreamArray(name='i_stream_array', dtype=dt_0)
j_stream_array = StreamArray(name='j_stream_array', dtype=dt_1)
def f_datatype(input_array):
output_array = np.zeros(len(input_array), dtype=dt_1)
output_array['time'] = input_array['time']
output_array['value'] = np.sum(input_array['value'], axis=1)
return output_array
e_np_agent = map_list(func=f_datatype,
in_stream=i_stream_array,
out_stream=j_stream_array,
name='e_np_agent')
c_array = np.array([(1, [2.0, 3.0, 4.0])], dtype=dt_0)
assert j_stream_array.stop == 0
i_stream_array.extend(c_array)
scheduler.step()
assert np.array_equal(j_stream_array.recent[:j_stream_array.stop],
f_datatype(c_array))
d_array = np.array([(10, [6.0, 7.0, 8.0]), (20, [10.0, 11.0, 12.0])],
dtype=dt_0)
i_stream_array.extend(d_array)
scheduler.step()
assert np.array_equal(j_stream_array.recent[:j_stream_array.stop],
f_datatype(np.hstack((c_array, d_array))))
# Test list map with a datatype and positive integer dimension.
k_stream_array = StreamArray(name='k_stream_array',
dtype=dt_0,
dimension=2)
l_stream_array = StreamArray(name='l_stream_array', dtype=dt_1)
def f_datatype_int_dimension(input_array):
m = len(input_array)
output_array = np.zeros(m, dtype=dt_1)
for i in range(m):
output_array[i]['time'] = np.max(input_array[i]['time'])
output_array[i]['value'] = np.sum(input_array[i]['value'])
return output_array
f_np_agent = map_list(func=f_datatype_int_dimension,
in_stream=k_stream_array,
out_stream=l_stream_array,
name='f_np_agent')
e_array = np.array([[(1, [2.0, 3.0, 4.0]), (2, [5.0, 6.0, 7.0])]],
dtype=dt_0)
assert l_stream_array.stop == 0
k_stream_array.extend(e_array)
scheduler.step()
assert np.array_equal(l_stream_array.recent[:l_stream_array.stop],
f_datatype_int_dimension(e_array))
f_array = np.array([[(3, [8.0, 9.0, 10.0]), (4, [11.0, 12.0, 13.0])],
[(5, [-1.0, 0.0, 1.0]), (6, [-2.0, 2.0, -2.0])]],
dtype=dt_0)
k_stream_array.extend(f_array)
scheduler.step()
assert np.array_equal(
l_stream_array.recent[:l_stream_array.stop],
f_datatype_int_dimension(np.vstack((e_array, f_array))))
# Test list map with a datatype and a dimension which is a tuple
m_stream_array = StreamArray(name='m_stream_array',
dtype=dt_0,
dimension=(2, 2))
n_stream_array = StreamArray(name='n_stream_array', dtype=dt_1)
g_np_agent = map_list(func=f_datatype_int_dimension,
in_stream=m_stream_array,
out_stream=n_stream_array,
name='g_np_agent')
assert n_stream_array.stop == 0
g_array = np.array(
[
# zeroth 2x2 array
[[(1, [2.0, 3.0, 4.0]), (2, [5.0, 6.0, 7.0])],
[(3, [8.0, 9.0, 10.0]), (4, [11.0, 12.0, 13.0])]],
# first 2x2 array
[[(5, [12.0, 13.0, 14.0]), (6, [15.0, 16.0, 17.0])],
[(7, [18.0, 19.0, 20.0]), (8, [21.0, 22.0, 23.0])]]
],
dtype=dt_0)
m_stream_array.extend(g_array)
scheduler.step()
assert np.array_equal(n_stream_array.recent[:n_stream_array.stop],
f_datatype_int_dimension(g_array))
h_array = np.array([[[(9, [0.0, 1.0, -1.0]), (10, [2.0, 2.0, -4.0])],
[(11, [80.0, -71.0, -9.0]), (15, [0.0, 0.0, 0.0])]]],
dtype=dt_0)
m_stream_array.extend(h_array)
scheduler.step()
assert np.array_equal(
n_stream_array.recent[:n_stream_array.stop],
f_datatype_int_dimension(np.vstack((g_array, h_array))))
# Test list merge with StreamArray and no dimension and no data type
a_in_0 = StreamArray(name='a_in_0')
a_in_1 = StreamArray(name='a_in_1')
a_out = StreamArray(name='a_out')
def a_merge(list_of_lists):
array_0, array_1 = list_of_lists
return array_0 + array_1
a_s_agent = merge_list(func=a_merge,
in_streams=[a_in_0, a_in_1],
out_stream=a_out,
name='a_s_agent')
assert a_out.stop == 0
#a_in_0.extend(np.array([1.0, 2.0, 3.0]))
a_in_0.extend(np.array([1.0, 2.0, 3.0]))
scheduler.step()
assert a_out.stop == 0
a_in_0.extend(np.array([4.0, 5.0, 6.0]))
scheduler.step()
assert a_out.stop == 0
a_in_1.extend(np.array([10.0, 20.0]))
scheduler.step()
assert np.array_equal(a_out.recent[:a_out.stop], np.array([11.0, 22.0]))
a_in_1.extend(np.array([30.0, 40.0]))
scheduler.step()
assert np.array_equal(a_out.recent[:a_out.stop],
np.array([11.0, 22.0, 33.0, 44.0]))
# Test list merge with StreamArray and no dimension and data type
a_in_dt_0 = StreamArray(name='a_in_dt_0', dtype=dt_0)
a_in_dt_1 = StreamArray(name='a_in_dt_1', dtype=dt_0)
a_out_dt = StreamArray(name='out', dtype=dt_0)
def a_merge_dtype(list_of_arrays):
input_array_0, input_array_1 = list_of_arrays
output_array = np.zeros(len(input_array_0), dtype=dt_0)
output_array['time'] = \
np.max((input_array_0['time'], input_array_1['time']), axis=0)
output_array['value'] = input_array_0['value'] + input_array_1['value']
return output_array
a_s_dt_agent = merge_list(func=a_merge_dtype,
in_streams=[a_in_dt_0, a_in_dt_1],
out_stream=a_out_dt,
name='a_s_dt_agent')
a_in_dt_0.extend(np.array([(1, [1.0, 2.0, 3.0])], dtype=dt_0))
scheduler.step()
assert a_out_dt.stop == 0
a_in_dt_1.extend(np.array([(2, [10.0, 20.0, 30.0])], dtype=dt_0))
scheduler.step()
assert np.array_equal(a_out_dt.recent[:a_out_dt.stop],
np.array([(2, [11.0, 22.0, 33.0])], dtype=dt_0))
a_in_dt_0.extend(
np.array([(5, [21.0, 23.0, 32.0]), (9, [27.0, 29.0, 31.0])],
dtype=dt_0))
scheduler.step()
assert np.array_equal(a_out_dt.recent[:a_out_dt.stop],
np.array([(2, [11.0, 22.0, 33.0])], dtype=dt_0))
a_in_dt_1.extend(
np.array([(6, [19.0, 17.0, 8.0]), (8, [13.0, 11.0, 9.0]),
(10, [3.0, 1.0, 5.0])],
dtype=dt_0))
scheduler.step()
assert np.array_equal(
a_out_dt.recent[:a_out_dt.stop],
np.array([(2, [11.0, 22.0, 33.0]), (6, [40.0, 40.0, 40.0]),
(9, [40.0, 40.0, 40.0])],
dtype=dt_0))
# Test list split with StreamArray and positive integer dimension and no data type
dim = 2
b_in = StreamArray(name='b_in', dimension=dim)
b_out_0 = StreamArray(name='b_out_0', dimension=dim)
b_out_1 = StreamArray(name='b_out_1')
def b_split(array_of_arrays):
length = len(array_of_arrays)
output_array_0 = np.zeros((
length,
dim,
))
output_array_1 = np.zeros(length)
for i in range(length):
input_array = array_of_arrays[i]
output_array_0[i] = np.array(
[[np.max(input_array),
np.min(input_array)]])
output_array_1[i] = np.array([np.sum(input_array)])
return output_array_0, output_array_1
b_split_agent = split_list(func=b_split,
in_stream=b_in,
out_streams=[b_out_0, b_out_1],
name='b_split_agent')
b_array_0 = np.array([[1.0, 9.0]])
b_in.extend(b_array_0)
scheduler.step()
assert np.array_equal(b_out_0.recent[:b_out_0.stop], np.array([[9.0,
1.0]]))
assert np.array_equal(b_out_1.recent[:b_out_1.stop], np.array([10.0]))
b_array_1 = np.array([[98.0, 2.0]])
b_in.extend(b_array_1)
scheduler.step()
assert np.array_equal(b_out_0.recent[:b_out_0.stop],
np.array([[9.0, 1.0], [98.0, 2.0]]))
assert np.array_equal(b_out_1.recent[:b_out_1.stop],
np.array([10.0, 100.0]))
b_array_3 = np.array([[10.0, 20.0], [3.0, 37.0], [55.0, 5.0]])
b_in.extend(b_array_3)
scheduler.step()
assert np.array_equal(
b_out_0.recent[:b_out_0.stop],
np.array([[9.0, 1.0], [98.0, 2.0], [20.0, 10.0], [37.0, 3.0],
[55.0, 5.0]]))
assert np.array_equal(b_out_1.recent[:b_out_1.stop],
np.array([10.0, 100.0, 30.0, 40.0, 60.0]))
# Test list many with StreamArray and no dimension and no data type
c_in_0 = StreamArray(name='c_in_0')
c_in_1 = StreamArray(name='c_in_1')
c_out_0 = StreamArray(name='c_out_0')
c_out_1 = StreamArray(name='c_out_1')
def c_many(list_of_arrays):
length = len(list_of_arrays)
input_array_0, input_array_1 = list_of_arrays
output_array_0 = np.zeros(length)
output_array_1 = np.zeros(length)
output_array_0 = input_array_0 + input_array_1
output_array_1 = input_array_0 - input_array_1
return [output_array_0, output_array_1]
c_multi_agent = multi_list(func=c_many,
in_streams=[c_in_0, c_in_1],
out_streams=[c_out_0, c_out_1],
name='c_multi_agent')
c_array_0_0 = np.arange(3.0) * 3
c_array_1_0 = np.arange(3.0)
c_in_0.extend(c_array_0_0)
scheduler.step()
c_in_1.extend(c_array_1_0)
scheduler.step()
assert np.array_equal(c_out_0.recent[:c_out_0.stop],
np.array([0.0, 4.0, 8.0]))
assert np.array_equal(c_out_1.recent[:c_out_1.stop],
np.array([0.0, 2.0, 4.0]))
c_array_0_1 = np.array([100.0])
c_array_1_1 = np.array([4.0, 5.0, 6.0])
c_in_0.extend(c_array_0_1)
c_in_1.extend(c_array_1_1)
scheduler.step()
assert np.array_equal(c_out_0.recent[:c_out_0.stop],
np.array([0.0, 4.0, 8.0, 104.0]))
assert np.array_equal(c_out_1.recent[:c_out_1.stop],
np.array([0.0, 2.0, 4.0, 96.0]))
## # Test list many with StreamArray and no dimension and no data type
## z_in_0 = StreamArray(name='z_in_0')
## z_in_1 = StreamArray(name='z_in_1')
## z_out_0 = StreamArray(name='z_out_0')
## z_out_1 = StreamArray(name='z_out_1')
## def execute_list_of_np_func(v, list_of_np_func):
## length = len(list_of_arrays)
## input_array_0, input_array_1 = list_of_arrays
## output_array_0 = np.zeros(length)
## output_array_1 = np.zeros(length)
## output_array_0 = input_array_0 + input_array_1
## output_array_1 = input_array_0 - input_array_1
## return [output_array_0, output_array_1]
# Test list many with StreamArray and positive integer dimension and no data type
dim = 2
d_in_0 = StreamArray(name='d_in_0', dimension=dim)
d_in_1 = StreamArray(name='d_in_1', dimension=dim)
d_out_0 = StreamArray(name='d_out_0', dimension=dim)
d_out_1 = StreamArray(name='d_out_1')
def d_many(list_of_arrays):
length = len(list_of_arrays)
input_array_0, input_array_1 = list_of_arrays
output_array_0 = input_array_0 + input_array_1
output_array_1 = np.array([np.sum(input_array_0 + input_array_1)])
return output_array_0, output_array_1
d_multi_agent = multi_list(func=d_many,
in_streams=[d_in_0, d_in_1],
out_streams=[d_out_0, d_out_1],
name='d_multi_agent')
d_array_0_0 = np.array([[1.0, 2.0]])
d_array_1_0 = np.array([[0.0, 10.0]])
d_in_0.extend(d_array_0_0)
scheduler.step()
d_in_1.extend(d_array_1_0)
scheduler.step()
assert np.array_equal(d_out_0.recent[:d_out_0.stop], np.array([[1.0,
12.0]]))
assert np.array_equal(d_out_1.recent[:d_out_1.stop], np.array([13.0]))
d_array_0_1 = np.array([[4.0, 8.0]])
d_array_1_1 = np.array([[2.0, 4.0]])
d_in_0.extend(d_array_0_1)
d_in_1.extend(d_array_1_1)
scheduler.step()
assert np.array_equal(d_out_0.recent[:d_out_0.stop],
np.array([[1.0, 12.0], [6.0, 12.0]]))
assert np.array_equal(d_out_1.recent[:d_out_1.stop], np.array([13.0,
18.0]))
d_array_0_2 = np.array([[20.0, 30.0], [40.0, 50.0]])
d_array_1_2 = np.array([[-10.0, -20.0]])
d_in_0.extend(d_array_0_2)
d_in_1.extend(d_array_1_2)
scheduler.step()
assert np.array_equal(d_out_0.recent[:d_out_0.stop],
np.array([[1.0, 12.0], [6.0, 12.0], [10.0, 10.0]]))
assert np.array_equal(d_out_1.recent[:d_out_1.stop],
np.array([13.0, 18.0, 20.0]))
# Test list many with StreamArray and tuple dimension and no data type
dim = (2, 2)
e_in_0 = StreamArray(name='e_in_0', dimension=dim)
e_in_1 = StreamArray(name='e_in_1', dimension=dim)
e_out_0 = StreamArray(name='e_out_0', dimension=dim)
e_out_1 = StreamArray(name='e_out_1')
def e_many(list_of_arrays):
input_array_0, input_array_1 = list_of_arrays
output_array_0 = input_array_0 + input_array_1
output_array_1 = \
np.array([np.sum(input_array_0[i]+ input_array_1[i])
for i in range(len(input_array_0))])
return output_array_0, output_array_1
e_multi_agent = multi_list(func=e_many,
in_streams=[e_in_0, e_in_1],
out_streams=[e_out_0, e_out_1],
name='e_multi_agent')
e_array_0_0 = np.array([[[10.0, 20.0], [30.0, 40.0]]])
e_in_0.extend(e_array_0_0)
e_array_1_0 = np.array([[[1.0, 2.0], [3.0, 4.0]]])
e_in_1.extend(e_array_1_0)
scheduler.step()
assert np.array_equal(e_out_0.recent[:e_out_0.stop],
np.array([[[11.0, 22.0], [33.0, 44.0]]]))
assert np.array_equal(e_out_1.recent[:e_out_1.stop], np.array([110.0]))
e_array_0_1 = np.array([[[11.0, 13.0], [17.0, 19.0]],
[[2.0, 4.0], [6.0, 8.0]]])
e_in_0.extend(e_array_0_1)
scheduler.step()
assert np.array_equal(e_out_0.recent[:e_out_0.stop],
np.array([[[11.0, 22.0], [33.0, 44.0]]]))
assert np.array_equal(e_out_1.recent[:e_out_1.stop], np.array([110.0]))
e_array_1_1 = np.array([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0,
8.0]]])
e_in_1.extend(e_array_1_1)
scheduler.step()
assert np.array_equal(
e_out_0.recent[:e_out_0.stop],
np.array([[[11.0, 22.0], [33.0, 44.0]], [[12.0, 15.0], [20.0, 23.0]],
[[7.0, 10.0], [13.0, 16.0]]]))
assert np.array_equal(e_out_1.recent[:e_out_1.stop],
np.array([110.0, 70.0, 46.0]))
e_array_1_2 = np.array([[[11.0, 12.0], [13.0, 14.0]],
[[15.0, 16.0], [17.0, 18.0]]])
e_in_1.extend(e_array_1_2)
scheduler.step()
e_array_0_2 = np.array([[[-10.0, -11.0], [12.0, 16.0]],
[[-14.0, -15.0], [-16.0, -17.0]]])
e_in_0.extend(e_array_0_2)
scheduler.step()
assert np.array_equal(
e_out_0.recent[:e_out_0.stop],
np.array([[[11.0, 22.0], [33.0, 44.0]], [[12.0, 15.0], [20.0, 23.0]],
[[7.0, 10.0], [13.0, 16.0]], [[1.0, 1.0], [25.0, 30.0]],
[[1.0, 1.0], [1.0, 1.0]]]))
assert np.array_equal(e_out_1.recent[:e_out_1.stop],
np.array([110.0, 70.0, 46.0, 57.0, 4.0]))
#------------------------------------------------------------------
#------------------------------------------------------------------
# Test args and kwargs
#------------------------------------------------------------------
#------------------------------------------------------------------
# Test map
def map_args(lst, multiplicand):
return [multiplicand * element for element in lst]
in_stream_map_args_stream = Stream('in_stream_map_args_stream')
out_stream_map_args_stream = Stream('out_stream_map_args_stream')
out_stream_map_kwargs_stream = Stream('out_stream_map_kwargs_stream')
map_args_agent = map_list(map_args, in_stream_map_args_stream,
out_stream_map_args_stream, None, None,
'map_args_agent', 2)
map_kwargs_agent = map_list(func=map_args,
in_stream=in_stream_map_args_stream,
out_stream=out_stream_map_kwargs_stream,
name='map_args_agent',
multiplicand=2)
scheduler.step()
assert out_stream_map_args_stream.recent[:out_stream_map_args_stream.stop] == \
[]
assert out_stream_map_kwargs_stream.recent[:out_stream_map_kwargs_stream.stop] == \
[]
in_stream_map_args_stream.extend(range(5))
scheduler.step()
assert out_stream_map_args_stream.recent[:out_stream_map_args_stream.stop] == \
[0, 2, 4, 6, 8]
assert out_stream_map_kwargs_stream.recent[:out_stream_map_kwargs_stream.stop] == \
[0, 2, 4, 6, 8]
in_stream_map_args_stream.append(5)
scheduler.step()
assert out_stream_map_args_stream.recent[:out_stream_map_args_stream.stop] == \
[0, 2, 4, 6, 8, 10]
assert out_stream_map_kwargs_stream.recent[:out_stream_map_kwargs_stream.stop] == \
[0, 2, 4, 6, 8, 10]
# Test list map on StreamArray (dimension is 0).
a_stream_array_args = StreamArray(name='a_stream_array_args')
b_stream_array_args = StreamArray(name='b_stream_array_args')
c_stream_array_args_kwargs = StreamArray(name='c_stream_array_args_kwargs')
def f_np_args(input_array_args, addend):
return input_array_args + addend
def f_np_args_kwargs(input_array_args_kwargs, multiplicand, addend):
return input_array_args_kwargs * multiplicand + addend
a_np_agent_args = map_list(f_np_args, a_stream_array_args,
b_stream_array_args, None, None,
'a_np_agent_args', 1)
a_np_agent_args_kwargs = map_list(f_np_args_kwargs,
a_stream_array_args,
c_stream_array_args_kwargs,
None,
None,
'a_np_agent_args_kwargs',
2,
addend=10)
scheduler.step()
assert np.array_equal(
b_stream_array_args.recent[:b_stream_array_args.stop], np.array([]))
assert np.array_equal(
c_stream_array_args_kwargs.recent[:c_stream_array_args_kwargs.stop],
np.array([]))
a_stream_array_args.extend(np.arange(5.0))
scheduler.step()
assert np.array_equal(
b_stream_array_args.recent[:b_stream_array_args.stop],
np.arange(5.0) + 1)
assert np.array_equal(
c_stream_array_args_kwargs.recent[:c_stream_array_args_kwargs.stop],
np.arange(5.0) * 2 + 10)
a_stream_array_args.extend(np.arange(5.0, 10.0, 1.0))
scheduler.step()
assert np.array_equal(
b_stream_array_args.recent[:b_stream_array_args.stop],
np.arange(10.0) + 1)
assert np.array_equal(
c_stream_array_args_kwargs.recent[:c_stream_array_args_kwargs.stop],
np.arange(10.0) * 2 + 10)
print 'TEST OF OP (LISTS) IS SUCCESSFUL'
def test2(): # SET PARAMETERS
# fs: sample rate
fs = 50
# ma: maximum amplitude
ma = 1
# ps: phase shift
ps = 0.0
# td: time duration
td = 10.0
# or: order
order = 2
lowcut = 1
highcut = 5
# GENERATE WAVES
# Generate streams of waves with different frequencies,
# amplitudes and phase shifts. Each wave is a pure
# frequency.
wave_data_low_frequency = generate_sine_wave(frequency=0.25,
max_amplitude=ma,
phase_shift=ps,
sample_rate=fs,
time_duration=td)
wave_data_medium_frequency = generate_sine_wave(frequency=2.5,
max_amplitude=ma,
phase_shift=ps,
sample_rate=fs,
time_duration=td)
wave_data_high_frequency = generate_sine_wave(frequency=15.0,
max_amplitude=ma,
phase_shift=ps,
sample_rate=fs,
time_duration=td)
# Generate a wave that is the sum of pure-frequency
# waves.
wave_data_combined_frequencies = (wave_data_low_frequency +
wave_data_medium_frequency +
wave_data_high_frequency)
# -----------------------------------------------------------
# TEST BANDPASS FIR FILTER
# -----------------------------------------------------------
x = StreamArray('x')
y = StreamArray('y')
## # Create a bandpass filter that operates on an input
## # stream x to produce the output stream y. This filter
## # uses firwin() from scipy.signal
## b = fir_bandpass(lowcut, highcut, fs)
## BP_FIR(b).filter_stream(in_stream=x, out_stream=y)
# Create a bandpass filter that operates on an input
# stream x to produce the output stream y.
bandpass_filter_stream(x, y, lowcut, highcut, fs)
# Feed the input to the filter with combined frequencies.
x.extend(wave_data_combined_frequencies)
# Run a step and plot output.
Stream.scheduler.step()
y.extend(wave_data_medium_frequency)
before_filtering_data = recent_values(x)
after_filtering_data = recent_values(y)
# Plot data
print 'PLOTTING FIR FILTER'
before_filtering_data = recent_values(x)
after_filtering_data = recent_values(y)
plt.figure(1)
plt.subplot(211)
plt.plot(before_filtering_data)
plt.subplot(212)
plt.plot(after_filtering_data)
plt.show()