def pitchshift_stream(sound_array, n, window_size=2**13, h=2**11):
"""
Changes the pitch of a sound by n semitones.
Notes
-----
This application has 2 sink_window agents and 3 streams x, y, z.
Stretch agent: The first agent gets input x and outputs y which
stretches the data in stream x. The stretching code is from Zulko,
pianoputer.
Speed up agent: The next agent gets input y and outputs z which
speeds up y by the specified factor. This agent interpolates the
data in y to the number of points determined by factor.
"""
factor = 2**(1.0 * n / 12.0)
f = 1.0/factor
# Declare streams
x = StreamArray('x', dtype=np.int16)
y = StreamArray('y', dtype=np.int16)
z = StreamArray('z', dtype=np.int16)
# Define the stretch agent
stretch_object = Stretch(
in_stream=x, out_stream=y, factor=factor,
window_size=window_size, h=h)
sink_window(
func=stretch_object.stretch, in_stream=x,
window_size=window_size+h, step_size=int(h*f))
# Define the speedup agent.
def f(window, out_stream):
indices = np.arange(0, window_size, factor)
out_stream.extend(
np.int16(np.interp(
indices, np.arange(window_size), window)))
sink_window(func=f, in_stream=y, window_size=window_size,
step_size=window_size, out_stream=z)
# Partition sound_array into sound bites. Extend the
# input with a sequence of sound bites and run each
# sound bite until the sound_array data is finished.
sound_bite_size = 2**14
for i in range(0, sound_array.size, sound_bite_size):
# sound_bite = sound_array[i:i+sound_bite_size]
x.extend(sound_array[i:i+sound_bite_size])
run()
# Process any data in sound_array that wasn't processed
# in the for loop.
x.extend(sound_array[i:])
# Return the result.
return z.recent[:z.stop]
def __init__(self, in_stream, out_stream, n_components, batch_size,
n_recompute, plotter):
self.in_stream = in_stream
self.out_stream = out_stream
self.n_components = n_components
self.batch_size = batch_size
self.n_recompute = n_recompute
self.history = incremental_buffer(self.n_recompute * self.batch_size)
self.ipca = IncrementalPCA(self.n_components, self.batch_size)
self.plotter = plotter
sink_window(self.f,
self.in_stream,
window_size=self.batch_size,
step_size=self.batch_size)
def test_window_agents():
scheduler = Stream.scheduler
q = Stream('q')
qq = Stream('qq')
r = Stream('r')
s = Stream('s')
t = Stream('t')
u = Stream('u')
v = Stream('v')
w = Stream('w')
x = Stream('x')
y = Stream('y')
z = Stream('z')
a = Stream('a')
b = Stream('b')
c = Stream('c')
yy = Stream('yy')
zz = Stream('zz')
#----------------------------------------------------------------
# Test simple window map agent with the same window size and step size
smap = map_window_f(func=sum, in_stream=r, window_size=4, step_size=4)
map_window(func=sum, in_stream=r, out_stream=s, window_size=4, step_size=4)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test simple window list agent with the same window size and step size
def f_map_window_list(lst):
return [max(lst)] * len(lst)
s_list = Stream('s list')
map_window_list(func=f_map_window_list,
in_stream=r,
out_stream=s_list,
window_size=4,
step_size=4)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with different window and step sizes
map_window(func=sum, in_stream=r, out_stream=t, window_size=3, step_size=2)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with a NumPy function
map_window(func=np.mean,
in_stream=r,
out_stream=q,
window_size=3,
step_size=2,
name='bb')
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with arguments
def map_with_args(window, addend):
return np.mean(window) + addend
map_window(func=map_with_args,
in_stream=r,
out_stream=qq,
window_size=3,
step_size=2,
addend=1)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with user-defined function and no state
map_window(func=lambda v: sum(v) + 1,
in_stream=r,
out_stream=u,
window_size=4,
step_size=4)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with state
def g(lst, state):
return sum(lst) + state, sum(lst) + state
map_window(func=g,
in_stream=r,
out_stream=v,
window_size=4,
step_size=4,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window merge agent with no state
def h(list_of_windows):
return sum([sum(window) for window in list_of_windows])
merge_window(func=h,
in_streams=[r, w],
out_stream=x,
window_size=3,
step_size=3)
merge_stream = merge_window_f(func=h,
in_streams=[r, w],
window_size=3,
step_size=3)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window merge agent with state
def h_with_state(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_with_state,
in_streams=[r, w],
out_stream=a,
window_size=3,
step_size=3,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window split agent with no state
def splt(window):
return sum(window), max(window)
split_window(func=splt,
in_stream=r,
out_streams=[y, z],
window_size=3,
step_size=3)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window split agent with state
def split_with_state(window, state):
return (sum(window) + state, max(window) + state), state + 1
split_window(func=split_with_state,
in_stream=r,
out_streams=[yy, zz],
window_size=3,
step_size=3,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window many-to-many with state and args
def func_multi_window_with_state_and_args(windows, state, cutoff):
return ((max(max(windows[0]), max(windows[1]), cutoff, state),
min(min(windows[0]), min(windows[1]), cutoff,
state)), state + 2)
multi_window(func=func_multi_window_with_state_and_args,
in_streams=[r, w],
out_streams=[b, c],
state=0,
window_size=3,
step_size=3,
cutoff=15)
multi_window_b, multi_window_c = multi_window_f(
func=func_multi_window_with_state_and_args,
in_streams=[r, w],
num_out_streams=2,
state=0,
window_size=3,
step_size=3,
cutoff=15)
#----------------------------------------------------------------
#----------------------------------------------------------------
r.extend(list(range(16)))
scheduler.step()
assert recent_values(r) == [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
]
assert recent_values(s) == [
0 + 1 + 2 + 3, 4 + 5 + 6 + 7, 8 + 9 + 10 + 11, 12 + 13 + 14 + 15
]
assert recent_values(smap) == recent_values(s)
assert recent_values(t) == [
0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12,
12 + 13 + 14
]
assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13]
assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14]
assert recent_values(u) == [
0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1,
12 + 13 + 14 + 15 + 1
]
assert recent_values(v) == [6, 28, 66, 120]
assert recent_values(w) == []
assert recent_values(x) == []
assert recent_values(merge_stream) == recent_values(x)
# y is sum of windows of r with window and step size of 3
assert recent_values(y) == [3, 12, 21, 30, 39]
assert recent_values(yy) == [3, 13, 23, 33, 43]
# y is max of windows of r with window and step size of 3
assert recent_values(z) == [2, 5, 8, 11, 14]
assert recent_values(zz) == [2, 6, 10, 14, 18]
assert recent_values(a) == []
assert recent_values(b) == []
assert recent_values(c) == []
#----------------------------------------------------------------
#----------------------------------------------------------------
# Step through the scheduler
#----------------------------------------------------------------
#----------------------------------------------------------------
w.extend([10, 12, 14, 16, 18])
scheduler.step()
assert recent_values(r) == [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
]
assert recent_values(s) == [
0 + 1 + 2 + 3, 4 + 5 + 6 + 7, 8 + 9 + 10 + 11, 12 + 13 + 14 + 15
]
assert recent_values(s_list) == [
3, 3, 3, 3, 7, 7, 7, 7, 11, 11, 11, 11, 15, 15, 15, 15
]
assert recent_values(smap) == recent_values(s)
assert recent_values(t) == [
0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12,
12 + 13 + 14
]
assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13]
assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14]
assert recent_values(u) == [
0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1,
12 + 13 + 14 + 15 + 1
]
assert recent_values(v) == [6, 28, 66, 120]
assert recent_values(w) == [10, 12, 14, 16, 18]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(y) == [3, 12, 21, 30, 39]
assert recent_values(yy) == [3, 13, 23, 33, 43]
assert recent_values(z) == [2, 5, 8, 11, 14]
assert recent_values(zz) == [2, 6, 10, 14, 18]
assert recent_values(a) == [39]
assert recent_values(b) == [15]
assert recent_values(c) == [0]
#----------------------------------------------------------------
r.extend([10, -10, 21, -20])
scheduler.step()
assert recent_values(s) == [6, 22, 38, 54, 1]
assert recent_values(s_list) == \
[3, 3, 3, 3, 7, 7, 7, 7, 11, 11, 11, 11, 15, 15, 15, 15, 21, 21, 21, 21]
assert recent_values(smap) == recent_values(s)
assert recent_values(t) == [
0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12,
12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21
]
assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7]
assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8]
assert recent_values(u) == [
0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1,
12 + 13 + 14 + 15 + 1, 10 + (-10) + 21 + (-20) + 1
]
assert recent_values(v) == [6, 28, 66, 120, 121]
assert recent_values(w) == [10, 12, 14, 16, 18]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)]
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(y) == [3, 12, 21, 30, 39, 15]
assert recent_values(yy) == [3, 13, 23, 33, 43, 20]
assert recent_values(z) == [2, 5, 8, 11, 14, 15]
assert recent_values(zz) == [2, 6, 10, 14, 18, 20]
assert recent_values(a) == [39]
assert recent_values(b) == [15]
assert recent_values(c) == [0]
#----------------------------------------------------------------
w.append(20)
scheduler.step()
assert recent_values(s) == [6, 22, 38, 54, 1]
assert recent_values(smap) == recent_values(s)
assert recent_values(t) == [
0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12,
12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21
]
assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7]
assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8]
assert recent_values(u) == [
0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1,
12 + 13 + 14 + 15 + 1, 10 + (-10) + 21 + (-20) + 1
]
assert recent_values(v) == [6, 28, 66, 120, 121]
assert recent_values(w) == [10, 12, 14, 16, 18, 20]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(y) == [3, 12, 21, 30, 39, 15]
assert recent_values(yy) == [3, 13, 23, 33, 43, 20]
assert recent_values(z) == [2, 5, 8, 11, 14, 15]
assert recent_values(zz) == [2, 6, 10, 14, 18, 20]
assert recent_values(a) == [39, 67]
assert recent_values(b) == [15, 20]
assert recent_values(multi_window_b) == recent_values(b)
assert recent_values(c) == [0, 2]
assert recent_values(multi_window_c) == recent_values(c)
#----------------------------------------------------------------
r.extend([-1, 1, 0])
scheduler.step()
assert recent_values(s) == [6, 22, 38, 54, 1]
assert recent_values(smap) == recent_values(s)
assert recent_values(t) == [
0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12,
12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21, 21 - 20 - 1, -1 + 1 + 0
]
assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7, 0, 0]
assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8, 1, 1]
assert recent_values(u) == [7, 23, 39, 55, 2]
assert recent_values(v) == [6, 28, 66, 120, 121]
assert recent_values(w) == [10, 12, 14, 16, 18, 20]
assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14),
(3 + 4 + 5) + (16 + 18 + 20)]
assert recent_values(merge_stream) == recent_values(x)
assert recent_values(y) == [3, 12, 21, 30, 39, 15, 0]
assert recent_values(yy) == [3, 13, 23, 33, 43, 20, 6]
assert recent_values(z) == [2, 5, 8, 11, 14, 15, 21]
assert recent_values(zz) == [2, 6, 10, 14, 18, 20, 27]
assert recent_values(a) == [39, 67]
assert recent_values(b) == [15, 20]
assert recent_values(multi_window_b) == recent_values(b)
assert recent_values(c) == [0, 2]
assert recent_values(multi_window_c) == recent_values(c)
#----------------------------------------------------------------
#----------------------------------------------------------------
# TEST WINDOW WITH STREAM ARRAY
#----------------------------------------------------------------
#----------------------------------------------------------------
# Simple linear arrays
x = StreamArray('x')
y = StreamArray('y')
#----------------------------------------------------------------
# Test window map agent with stream arrays and a NumPy function
map_window(func=np.mean,
in_stream=x,
out_stream=y,
window_size=3,
step_size=3,
name='window map agent for arrays')
#----------------------------------------------------------------
#----------------------------------------------------------------
x.extend(np.linspace(0.0, 11.0, 12))
scheduler.step()
assert np.array_equal(recent_values(x), np.linspace(0.0, 11.0, 12))
# y[0] = (0+1+2)//3.0, y[1] = (3+4+5)//3.0
assert np.array_equal(recent_values(y), np.array([1.0, 4.0, 7.0, 10.0]))
x = StreamArray('x', dimension=2)
y = StreamArray('y', dimension=2)
z = StreamArray('z', dimension=2)
a = StreamArray('a', dimension=2)
b = StreamArray('b', dimension=2)
c = StreamArray('c', dimension=2)
d = StreamArray('d', dimension=2)
p = StreamArray('p', dimension=2)
q = StreamArray('q', dimension=2)
r = StreamArray('r', dimension=2)
s = StreamArray('s', dimension=2)
t = StreamArray('t', dimension=2)
#----------------------------------------------------------------
# Test window map agent with stream arrays and a NumPy function
# f() and ff() differ only in the axis.
def f(input_array):
return np.mean(input_array, axis=0)
def ff(input_array):
return np.mean(input_array, axis=1)
map_window(func=f,
in_stream=x,
out_stream=y,
window_size=2,
step_size=2,
name='window map agent for arrays')
map_window(func=ff,
in_stream=x,
out_stream=t,
window_size=2,
step_size=2,
name='window map agent for arrays ff')
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window sink with stream arrays
def sum_array(input_array, output_list):
output_list.append(sum(input_array))
sum_array_list = []
sink_window(func=sum_array,
in_stream=x,
window_size=2,
step_size=2,
name='sum array',
output_list=sum_array_list)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window map agent with state
def g(lst, state):
return sum(lst) + state, state + 1
map_window(func=g,
in_stream=x,
out_stream=z,
window_size=2,
step_size=2,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window merge agent with state
def h_array(list_of_windows, state):
return (sum([sum(window)
for window in list_of_windows]) + state, state + 1)
merge_window(func=h_array,
in_streams=[x, a],
out_stream=b,
window_size=2,
step_size=2,
state=0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window split agent with state
def split_with_state(window, state):
return [np.sum(window, axis=0)+state, np.max(window, axis=0)+state], \
state+1.0
split_window(func=split_with_state,
in_stream=x,
out_streams=[c, d],
window_size=2,
step_size=2,
state=0.0)
#----------------------------------------------------------------
#----------------------------------------------------------------
# Test window many-to-many with state and args
def func_multi_window_with_state_and_args(windows, state, cutoff):
max_value = np.maximum(np.max(windows[0], axis=0),
np.max(windows[1], axis=0))
max_value = np.maximum(max_value, cutoff) + state
min_value = np.minimum(np.min(windows[0], axis=0),
np.min(windows[1], axis=0))
min_value = np.minimum(min_value, cutoff) + state
return (max_value, min_value), state + 1
multi_window(func=func_multi_window_with_state_and_args,
in_streams=[x, a],
out_streams=[r, s],
state=0,
window_size=2,
step_size=2,
cutoff=10)
#----------------------------------------------------------------
x.extend(np.array([[1., 5.], [7., 11.]]))
a.extend(np.array([[0., 1.], [2., 3.]]))
scheduler.step()
# sum_array_list is the sum of x with window size, step size of 2
assert np.array_equal(sum_array_list, [np.array([1. + 7., 5. + 11.])])
# y is the mean of x with window size and step size of 2
assert np.array_equal(recent_values(x), np.array([[1., 5.], [7., 11.]]))
assert np.array_equal(recent_values(y),
np.array([[(1. + 7.) // 2.0, (5. + 11.) // 2.]]))
assert np.array_equal(recent_values(t),
np.array([[(1. + 5.) // 2.0, (7. + 11.) // 2.]]))
assert np.array_equal(recent_values(z), np.array([[1. + 7., 5. + 11]]))
assert np.array_equal(recent_values(b),
[np.array([1. + 7. + 0. + 2., 5. + 11. + 1. + 3.])])
assert np.array_equal(recent_values(c), np.array([[8., 16.]]))
assert np.array_equal(recent_values(d), np.array([[7., 11.]]))
assert np.array_equal(recent_values(r), np.array([[10., 11.]]))
assert np.array_equal(recent_values(s), np.array([[0., 1.]]))
a.extend(np.array([[0., 1.], [1., 0.]]))
scheduler.step()
assert np.array_equal(recent_values(y),
np.array([[(1. + 7.) / 2.0, (5. + 11.) / 2.]]))
assert np.array_equal(recent_values(z), np.array([[1. + 7., 5. + 11]]))
assert np.array_equal(recent_values(b),
[np.array([1. + 7. + 0. + 2., 5. + 11. + 1. + 3.])])
assert np.array_equal(recent_values(c), np.array([[8., 16.]]))
assert np.array_equal(recent_values(d), np.array([[7., 11.]]))
assert np.array_equal(recent_values(r), np.array([[10., 11.]]))
assert np.array_equal(recent_values(s), np.array([[0., 1.]]))
x.extend(np.array([[14., 18.], [18., 30.], [30., 38.], [34., 42.]]))
scheduler.step()
assert np.array_equal(recent_values(y),
np.array([[4., 8.], [16., 24.], [32., 40.]]))
assert np.array_equal(recent_values(z),
np.array([[8., 16.], [33., 49.], [66., 82.]]))
assert np.array_equal(recent_values(c),
np.array([[8., 16.], [33., 49.], [66., 82.]]))
assert np.array_equal(recent_values(d),
np.array([[7., 11.], [19., 31.], [36., 44.]]))
assert np.array_equal(recent_values(r), np.array([[10., 11.], [19., 31.]]))
assert np.array_equal(recent_values(s), np.array([[0., 1.], [1., 1.]]))
print('TEST OF OP (WINDOW) IS SUCCESSFUL')
return
def g(in_stream, window_size, step_size, **kwargs):
sink_window(func, in_stream, window_size, step_size, **kwargs)
def compute_func(in_streams, out_streams):
def partial_fit(X, state):
"""Incremental fit with X. All of X is processed as a single batch.
Parameters
----------
X : array-like, shape (n_samples, n_features)
Training data, where n_samples is the number of samples and
n_features is the number of features.
Returns
-------
self : object
Returns the instance itself.
"""
end = False
print(X[0])
if X[-1] == 'eof':
end = True
X = X[:-1]
X = np.array(X, dtype='float64')
n_samples, n_features = X.shape
state.count += n_samples
# Update stats - they are 0 if this is the fisrt step
col_mean, col_var, n_total_samples = \
_incremental_mean_and_var(
X, last_mean=state.mean_, last_variance=state.var_,
last_sample_count=np.repeat(state.n_samples_seen_, X.shape[1]))
n_total_samples = n_total_samples[0]
# Whitening
if state.n_samples_seen_ == 0:
# If it is the first step, simply whiten X
X -= col_mean
else:
col_batch_mean = np.mean(X, axis=0)
X -= col_batch_mean
# Build matrix of combined previous basis and new data
mean_correction = \
np.sqrt((state.n_samples_seen_ * n_samples) /
n_total_samples) * (state.mean_ - col_batch_mean)
X = np.vstack((state.singular_values_.reshape(
(-1, 1)) * state.components_, X, mean_correction))
U, S, V = linalg.svd(X, full_matrices=False)
U, V = svd_flip(U, V, u_based_decision=False)
explained_variance = S**2 / (n_total_samples - 1)
explained_variance_ratio = S**2 / np.sum(col_var * n_total_samples)
state.n_samples_seen_ = n_total_samples
state.components_ = V[:state.n_components_]
state.singular_values_ = S[:state.n_components_]
state.mean_ = col_mean
state.var_ = col_var
state.explained_variance_ = explained_variance[:state.
n_components_]
state.explained_variance_ratio_ = \
explained_variance_ratio[:state.n_components_]
if state.n_components_ < n_features:
state.noise_variance_ = \
explained_variance[state.n_components_:].mean()
else:
state.noise_variance_ = 0.
if state.count >= 999:
state.count = 0
print(state.components_)
if end:
np.savetxt("IPCA.dat", state.components_)
return state
sink_window(func=partial_fit,
in_stream=in_streams[0],
window_size=batches,
step_size=batches,
state=state)
def compute_func(in_streams, out_streams):
# IDENTIFY INPUT AND OUTPUT STREAMS.
# Original stream list = in_streams[0]
# CREATE INTERNAL STREAMS.
# Internal Stream - out
# It is used to output the top-k items and their counts after fixed intervals
# Each element of the stream is a list of 2-sized tuples where,
# tuple[1] = item
# tuple[0] = its count
#
out = Stream()
def count(counter, h, p, v):
"""
Given the counter and the hash functions, this function returns the count of the item
Attributes
----------
counter: 2-d array
Stores the hash bucket values
h: 2-dimensional list of ints
defines the hash function used to decide which bucket the count of v should be added to
p: int
the prime number used to create both the hash functions
v: int
An element of the stream
Returns
--------
co: int
Count of the input item
"""
co = -1
for i in range(counter.shape[0]):
v_bin = (v**2) * h[i][0] + v * h[i][1] + h[i][2]
v_bin = v_bin % p % b
temp = counter[i][v_bin]
if co == -1 or temp < co:
co = temp
return int(co)
#DEFINE THE COUNTER UPDATER
#Updates the counter for every value in the stream
def counter_update(v, state, h, p, k):
"""
This function updates the counter for a given value.
If it sees the end of a finite stream it writes (pickles) the heap of top-k items
into the file CMS.dat.
Attributes
----------
v: int
An element of the stream
state: multidimensional list
state stores the hash bucket (called counter), number of elements seen and the heap containing the top-k elements
counter = state[0]: 2-d array
number of elements = state[1]: int
heap: = state[2]: array of 2-element tuples
h: 2-dimensional list of ints
defines the hash function used to decide which bucket the count of v should be added to
p: int
the prime number used to create both the hash functions
k: int
Specifies the number of heavy hitter elements
"""
counter = state[0]
heap = state[2]
if v == 'eof':
heap = np.array(heap)
np.savetxt("CMS.dat", heap)
return state[2], state
state[1] += 1
for i in range(counter.shape[0]):
v_bin = (v**2) * h[i][0] + v * h[i][1] + h[i][2]
v_bin = v_bin % p % b
counter[i][v_bin] += 1
v_count = count(counter, h, p, v)
if v_count >= state[1] / k:
for i in range(len(heap)):
if v == heap[i][1]:
heap[i] = (v_count, v)
heapq.heapify(heap)
break
else:
heapq.heappush(heap, (v_count, v))
while len(heap) > k:
heapq.heappop(heap)
state[0] = counter
state[2] = heap
return state[2], state
def regular_print(v):
""" This function prints the last value of a window of the input stream
"""
print v[-1]
# CREATE AGENTS
# Make the agent that applies the counter update
# function to the input stream
map_element(func=counter_update,
in_stream=in_streams[0],
out_stream=out,
state=state,
h=h,
p=p,
k=k)
#Make the agent which prints the counter stream at regular intervals
sink_window(func=regular_print,
in_stream=out,
window_size=printer,
step_size=printer)