def pitchshift_stream(sound_array, n, window_size=2**13, h=2**11): """ Changes the pitch of a sound by n semitones. Notes ----- This application has 2 sink_window agents and 3 streams x, y, z. Stretch agent: The first agent gets input x and outputs y which stretches the data in stream x. The stretching code is from Zulko, pianoputer. Speed up agent: The next agent gets input y and outputs z which speeds up y by the specified factor. This agent interpolates the data in y to the number of points determined by factor. """ factor = 2**(1.0 * n / 12.0) f = 1.0/factor # Declare streams x = StreamArray('x', dtype=np.int16) y = StreamArray('y', dtype=np.int16) z = StreamArray('z', dtype=np.int16) # Define the stretch agent stretch_object = Stretch( in_stream=x, out_stream=y, factor=factor, window_size=window_size, h=h) sink_window( func=stretch_object.stretch, in_stream=x, window_size=window_size+h, step_size=int(h*f)) # Define the speedup agent. def f(window, out_stream): indices = np.arange(0, window_size, factor) out_stream.extend( np.int16(np.interp( indices, np.arange(window_size), window))) sink_window(func=f, in_stream=y, window_size=window_size, step_size=window_size, out_stream=z) # Partition sound_array into sound bites. Extend the # input with a sequence of sound bites and run each # sound bite until the sound_array data is finished. sound_bite_size = 2**14 for i in range(0, sound_array.size, sound_bite_size): # sound_bite = sound_array[i:i+sound_bite_size] x.extend(sound_array[i:i+sound_bite_size]) run() # Process any data in sound_array that wasn't processed # in the for loop. x.extend(sound_array[i:]) # Return the result. return z.recent[:z.stop]
def __init__(self, in_stream, out_stream, n_components, batch_size, n_recompute, plotter): self.in_stream = in_stream self.out_stream = out_stream self.n_components = n_components self.batch_size = batch_size self.n_recompute = n_recompute self.history = incremental_buffer(self.n_recompute * self.batch_size) self.ipca = IncrementalPCA(self.n_components, self.batch_size) self.plotter = plotter sink_window(self.f, self.in_stream, window_size=self.batch_size, step_size=self.batch_size)
def test_window_agents(): scheduler = Stream.scheduler q = Stream('q') qq = Stream('qq') r = Stream('r') s = Stream('s') t = Stream('t') u = Stream('u') v = Stream('v') w = Stream('w') x = Stream('x') y = Stream('y') z = Stream('z') a = Stream('a') b = Stream('b') c = Stream('c') yy = Stream('yy') zz = Stream('zz') #---------------------------------------------------------------- # Test simple window map agent with the same window size and step size smap = map_window_f(func=sum, in_stream=r, window_size=4, step_size=4) map_window(func=sum, in_stream=r, out_stream=s, window_size=4, step_size=4) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test simple window list agent with the same window size and step size def f_map_window_list(lst): return [max(lst)] * len(lst) s_list = Stream('s list') map_window_list(func=f_map_window_list, in_stream=r, out_stream=s_list, window_size=4, step_size=4) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with different window and step sizes map_window(func=sum, in_stream=r, out_stream=t, window_size=3, step_size=2) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with a NumPy function map_window(func=np.mean, in_stream=r, out_stream=q, window_size=3, step_size=2, name='bb') #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with arguments def map_with_args(window, addend): return np.mean(window) + addend map_window(func=map_with_args, in_stream=r, out_stream=qq, window_size=3, step_size=2, addend=1) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with user-defined function and no state map_window(func=lambda v: sum(v) + 1, in_stream=r, out_stream=u, window_size=4, step_size=4) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with state def g(lst, state): return sum(lst) + state, sum(lst) + state map_window(func=g, in_stream=r, out_stream=v, window_size=4, step_size=4, state=0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window merge agent with no state def h(list_of_windows): return sum([sum(window) for window in list_of_windows]) merge_window(func=h, in_streams=[r, w], out_stream=x, window_size=3, step_size=3) merge_stream = merge_window_f(func=h, in_streams=[r, w], window_size=3, step_size=3) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window merge agent with state def h_with_state(list_of_windows, state): return (sum([sum(window) for window in list_of_windows]) + state, state + 1) merge_window(func=h_with_state, in_streams=[r, w], out_stream=a, window_size=3, step_size=3, state=0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window split agent with no state def splt(window): return sum(window), max(window) split_window(func=splt, in_stream=r, out_streams=[y, z], window_size=3, step_size=3) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window split agent with state def split_with_state(window, state): return (sum(window) + state, max(window) + state), state + 1 split_window(func=split_with_state, in_stream=r, out_streams=[yy, zz], window_size=3, step_size=3, state=0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window many-to-many with state and args def func_multi_window_with_state_and_args(windows, state, cutoff): return ((max(max(windows[0]), max(windows[1]), cutoff, state), min(min(windows[0]), min(windows[1]), cutoff, state)), state + 2) multi_window(func=func_multi_window_with_state_and_args, in_streams=[r, w], out_streams=[b, c], state=0, window_size=3, step_size=3, cutoff=15) multi_window_b, multi_window_c = multi_window_f( func=func_multi_window_with_state_and_args, in_streams=[r, w], num_out_streams=2, state=0, window_size=3, step_size=3, cutoff=15) #---------------------------------------------------------------- #---------------------------------------------------------------- r.extend(list(range(16))) scheduler.step() assert recent_values(r) == [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] assert recent_values(s) == [ 0 + 1 + 2 + 3, 4 + 5 + 6 + 7, 8 + 9 + 10 + 11, 12 + 13 + 14 + 15 ] assert recent_values(smap) == recent_values(s) assert recent_values(t) == [ 0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12, 12 + 13 + 14 ] assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13] assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14] assert recent_values(u) == [ 0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1, 12 + 13 + 14 + 15 + 1 ] assert recent_values(v) == [6, 28, 66, 120] assert recent_values(w) == [] assert recent_values(x) == [] assert recent_values(merge_stream) == recent_values(x) # y is sum of windows of r with window and step size of 3 assert recent_values(y) == [3, 12, 21, 30, 39] assert recent_values(yy) == [3, 13, 23, 33, 43] # y is max of windows of r with window and step size of 3 assert recent_values(z) == [2, 5, 8, 11, 14] assert recent_values(zz) == [2, 6, 10, 14, 18] assert recent_values(a) == [] assert recent_values(b) == [] assert recent_values(c) == [] #---------------------------------------------------------------- #---------------------------------------------------------------- # Step through the scheduler #---------------------------------------------------------------- #---------------------------------------------------------------- w.extend([10, 12, 14, 16, 18]) scheduler.step() assert recent_values(r) == [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ] assert recent_values(s) == [ 0 + 1 + 2 + 3, 4 + 5 + 6 + 7, 8 + 9 + 10 + 11, 12 + 13 + 14 + 15 ] assert recent_values(s_list) == [ 3, 3, 3, 3, 7, 7, 7, 7, 11, 11, 11, 11, 15, 15, 15, 15 ] assert recent_values(smap) == recent_values(s) assert recent_values(t) == [ 0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12, 12 + 13 + 14 ] assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13] assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14] assert recent_values(u) == [ 0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1, 12 + 13 + 14 + 15 + 1 ] assert recent_values(v) == [6, 28, 66, 120] assert recent_values(w) == [10, 12, 14, 16, 18] assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)] assert recent_values(merge_stream) == recent_values(x) assert recent_values(y) == [3, 12, 21, 30, 39] assert recent_values(yy) == [3, 13, 23, 33, 43] assert recent_values(z) == [2, 5, 8, 11, 14] assert recent_values(zz) == [2, 6, 10, 14, 18] assert recent_values(a) == [39] assert recent_values(b) == [15] assert recent_values(c) == [0] #---------------------------------------------------------------- r.extend([10, -10, 21, -20]) scheduler.step() assert recent_values(s) == [6, 22, 38, 54, 1] assert recent_values(s_list) == \ [3, 3, 3, 3, 7, 7, 7, 7, 11, 11, 11, 11, 15, 15, 15, 15, 21, 21, 21, 21] assert recent_values(smap) == recent_values(s) assert recent_values(t) == [ 0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12, 12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21 ] assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7] assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8] assert recent_values(u) == [ 0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1, 12 + 13 + 14 + 15 + 1, 10 + (-10) + 21 + (-20) + 1 ] assert recent_values(v) == [6, 28, 66, 120, 121] assert recent_values(w) == [10, 12, 14, 16, 18] assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14)] assert recent_values(merge_stream) == recent_values(x) assert recent_values(y) == [3, 12, 21, 30, 39, 15] assert recent_values(yy) == [3, 13, 23, 33, 43, 20] assert recent_values(z) == [2, 5, 8, 11, 14, 15] assert recent_values(zz) == [2, 6, 10, 14, 18, 20] assert recent_values(a) == [39] assert recent_values(b) == [15] assert recent_values(c) == [0] #---------------------------------------------------------------- w.append(20) scheduler.step() assert recent_values(s) == [6, 22, 38, 54, 1] assert recent_values(smap) == recent_values(s) assert recent_values(t) == [ 0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12, 12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21 ] assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7] assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8] assert recent_values(u) == [ 0 + 1 + 2 + 3 + 1, 4 + 5 + 6 + 7 + 1, 8 + 9 + 10 + 11 + 1, 12 + 13 + 14 + 15 + 1, 10 + (-10) + 21 + (-20) + 1 ] assert recent_values(v) == [6, 28, 66, 120, 121] assert recent_values(w) == [10, 12, 14, 16, 18, 20] assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14), (3 + 4 + 5) + (16 + 18 + 20)] assert recent_values(merge_stream) == recent_values(x) assert recent_values(y) == [3, 12, 21, 30, 39, 15] assert recent_values(yy) == [3, 13, 23, 33, 43, 20] assert recent_values(z) == [2, 5, 8, 11, 14, 15] assert recent_values(zz) == [2, 6, 10, 14, 18, 20] assert recent_values(a) == [39, 67] assert recent_values(b) == [15, 20] assert recent_values(multi_window_b) == recent_values(b) assert recent_values(c) == [0, 2] assert recent_values(multi_window_c) == recent_values(c) #---------------------------------------------------------------- r.extend([-1, 1, 0]) scheduler.step() assert recent_values(s) == [6, 22, 38, 54, 1] assert recent_values(smap) == recent_values(s) assert recent_values(t) == [ 0 + 1 + 2, 2 + 3 + 4, 4 + 5 + 6, 6 + 7 + 8, 8 + 9 + 10, 10 + 11 + 12, 12 + 13 + 14, 14 + 15 + 10, 10 + (-10) + 21, 21 - 20 - 1, -1 + 1 + 0 ] assert recent_values(q) == [1, 3, 5, 7, 9, 11, 13, 13, 7, 0, 0] assert recent_values(qq) == [2, 4, 6, 8, 10, 12, 14, 14, 8, 1, 1] assert recent_values(u) == [7, 23, 39, 55, 2] assert recent_values(v) == [6, 28, 66, 120, 121] assert recent_values(w) == [10, 12, 14, 16, 18, 20] assert recent_values(x) == [(0 + 1 + 2) + (10 + 12 + 14), (3 + 4 + 5) + (16 + 18 + 20)] assert recent_values(merge_stream) == recent_values(x) assert recent_values(y) == [3, 12, 21, 30, 39, 15, 0] assert recent_values(yy) == [3, 13, 23, 33, 43, 20, 6] assert recent_values(z) == [2, 5, 8, 11, 14, 15, 21] assert recent_values(zz) == [2, 6, 10, 14, 18, 20, 27] assert recent_values(a) == [39, 67] assert recent_values(b) == [15, 20] assert recent_values(multi_window_b) == recent_values(b) assert recent_values(c) == [0, 2] assert recent_values(multi_window_c) == recent_values(c) #---------------------------------------------------------------- #---------------------------------------------------------------- # TEST WINDOW WITH STREAM ARRAY #---------------------------------------------------------------- #---------------------------------------------------------------- # Simple linear arrays x = StreamArray('x') y = StreamArray('y') #---------------------------------------------------------------- # Test window map agent with stream arrays and a NumPy function map_window(func=np.mean, in_stream=x, out_stream=y, window_size=3, step_size=3, name='window map agent for arrays') #---------------------------------------------------------------- #---------------------------------------------------------------- x.extend(np.linspace(0.0, 11.0, 12)) scheduler.step() assert np.array_equal(recent_values(x), np.linspace(0.0, 11.0, 12)) # y[0] = (0+1+2)//3.0, y[1] = (3+4+5)//3.0 assert np.array_equal(recent_values(y), np.array([1.0, 4.0, 7.0, 10.0])) x = StreamArray('x', dimension=2) y = StreamArray('y', dimension=2) z = StreamArray('z', dimension=2) a = StreamArray('a', dimension=2) b = StreamArray('b', dimension=2) c = StreamArray('c', dimension=2) d = StreamArray('d', dimension=2) p = StreamArray('p', dimension=2) q = StreamArray('q', dimension=2) r = StreamArray('r', dimension=2) s = StreamArray('s', dimension=2) t = StreamArray('t', dimension=2) #---------------------------------------------------------------- # Test window map agent with stream arrays and a NumPy function # f() and ff() differ only in the axis. def f(input_array): return np.mean(input_array, axis=0) def ff(input_array): return np.mean(input_array, axis=1) map_window(func=f, in_stream=x, out_stream=y, window_size=2, step_size=2, name='window map agent for arrays') map_window(func=ff, in_stream=x, out_stream=t, window_size=2, step_size=2, name='window map agent for arrays ff') #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window sink with stream arrays def sum_array(input_array, output_list): output_list.append(sum(input_array)) sum_array_list = [] sink_window(func=sum_array, in_stream=x, window_size=2, step_size=2, name='sum array', output_list=sum_array_list) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window map agent with state def g(lst, state): return sum(lst) + state, state + 1 map_window(func=g, in_stream=x, out_stream=z, window_size=2, step_size=2, state=0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window merge agent with state def h_array(list_of_windows, state): return (sum([sum(window) for window in list_of_windows]) + state, state + 1) merge_window(func=h_array, in_streams=[x, a], out_stream=b, window_size=2, step_size=2, state=0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window split agent with state def split_with_state(window, state): return [np.sum(window, axis=0)+state, np.max(window, axis=0)+state], \ state+1.0 split_window(func=split_with_state, in_stream=x, out_streams=[c, d], window_size=2, step_size=2, state=0.0) #---------------------------------------------------------------- #---------------------------------------------------------------- # Test window many-to-many with state and args def func_multi_window_with_state_and_args(windows, state, cutoff): max_value = np.maximum(np.max(windows[0], axis=0), np.max(windows[1], axis=0)) max_value = np.maximum(max_value, cutoff) + state min_value = np.minimum(np.min(windows[0], axis=0), np.min(windows[1], axis=0)) min_value = np.minimum(min_value, cutoff) + state return (max_value, min_value), state + 1 multi_window(func=func_multi_window_with_state_and_args, in_streams=[x, a], out_streams=[r, s], state=0, window_size=2, step_size=2, cutoff=10) #---------------------------------------------------------------- x.extend(np.array([[1., 5.], [7., 11.]])) a.extend(np.array([[0., 1.], [2., 3.]])) scheduler.step() # sum_array_list is the sum of x with window size, step size of 2 assert np.array_equal(sum_array_list, [np.array([1. + 7., 5. + 11.])]) # y is the mean of x with window size and step size of 2 assert np.array_equal(recent_values(x), np.array([[1., 5.], [7., 11.]])) assert np.array_equal(recent_values(y), np.array([[(1. + 7.) // 2.0, (5. + 11.) // 2.]])) assert np.array_equal(recent_values(t), np.array([[(1. + 5.) // 2.0, (7. + 11.) // 2.]])) assert np.array_equal(recent_values(z), np.array([[1. + 7., 5. + 11]])) assert np.array_equal(recent_values(b), [np.array([1. + 7. + 0. + 2., 5. + 11. + 1. + 3.])]) assert np.array_equal(recent_values(c), np.array([[8., 16.]])) assert np.array_equal(recent_values(d), np.array([[7., 11.]])) assert np.array_equal(recent_values(r), np.array([[10., 11.]])) assert np.array_equal(recent_values(s), np.array([[0., 1.]])) a.extend(np.array([[0., 1.], [1., 0.]])) scheduler.step() assert np.array_equal(recent_values(y), np.array([[(1. + 7.) / 2.0, (5. + 11.) / 2.]])) assert np.array_equal(recent_values(z), np.array([[1. + 7., 5. + 11]])) assert np.array_equal(recent_values(b), [np.array([1. + 7. + 0. + 2., 5. + 11. + 1. + 3.])]) assert np.array_equal(recent_values(c), np.array([[8., 16.]])) assert np.array_equal(recent_values(d), np.array([[7., 11.]])) assert np.array_equal(recent_values(r), np.array([[10., 11.]])) assert np.array_equal(recent_values(s), np.array([[0., 1.]])) x.extend(np.array([[14., 18.], [18., 30.], [30., 38.], [34., 42.]])) scheduler.step() assert np.array_equal(recent_values(y), np.array([[4., 8.], [16., 24.], [32., 40.]])) assert np.array_equal(recent_values(z), np.array([[8., 16.], [33., 49.], [66., 82.]])) assert np.array_equal(recent_values(c), np.array([[8., 16.], [33., 49.], [66., 82.]])) assert np.array_equal(recent_values(d), np.array([[7., 11.], [19., 31.], [36., 44.]])) assert np.array_equal(recent_values(r), np.array([[10., 11.], [19., 31.]])) assert np.array_equal(recent_values(s), np.array([[0., 1.], [1., 1.]])) print('TEST OF OP (WINDOW) IS SUCCESSFUL') return
def g(in_stream, window_size, step_size, **kwargs): sink_window(func, in_stream, window_size, step_size, **kwargs)
def compute_func(in_streams, out_streams): def partial_fit(X, state): """Incremental fit with X. All of X is processed as a single batch. Parameters ---------- X : array-like, shape (n_samples, n_features) Training data, where n_samples is the number of samples and n_features is the number of features. Returns ------- self : object Returns the instance itself. """ end = False print(X[0]) if X[-1] == 'eof': end = True X = X[:-1] X = np.array(X, dtype='float64') n_samples, n_features = X.shape state.count += n_samples # Update stats - they are 0 if this is the fisrt step col_mean, col_var, n_total_samples = \ _incremental_mean_and_var( X, last_mean=state.mean_, last_variance=state.var_, last_sample_count=np.repeat(state.n_samples_seen_, X.shape[1])) n_total_samples = n_total_samples[0] # Whitening if state.n_samples_seen_ == 0: # If it is the first step, simply whiten X X -= col_mean else: col_batch_mean = np.mean(X, axis=0) X -= col_batch_mean # Build matrix of combined previous basis and new data mean_correction = \ np.sqrt((state.n_samples_seen_ * n_samples) / n_total_samples) * (state.mean_ - col_batch_mean) X = np.vstack((state.singular_values_.reshape( (-1, 1)) * state.components_, X, mean_correction)) U, S, V = linalg.svd(X, full_matrices=False) U, V = svd_flip(U, V, u_based_decision=False) explained_variance = S**2 / (n_total_samples - 1) explained_variance_ratio = S**2 / np.sum(col_var * n_total_samples) state.n_samples_seen_ = n_total_samples state.components_ = V[:state.n_components_] state.singular_values_ = S[:state.n_components_] state.mean_ = col_mean state.var_ = col_var state.explained_variance_ = explained_variance[:state. n_components_] state.explained_variance_ratio_ = \ explained_variance_ratio[:state.n_components_] if state.n_components_ < n_features: state.noise_variance_ = \ explained_variance[state.n_components_:].mean() else: state.noise_variance_ = 0. if state.count >= 999: state.count = 0 print(state.components_) if end: np.savetxt("IPCA.dat", state.components_) return state sink_window(func=partial_fit, in_stream=in_streams[0], window_size=batches, step_size=batches, state=state)
def compute_func(in_streams, out_streams): # IDENTIFY INPUT AND OUTPUT STREAMS. # Original stream list = in_streams[0] # CREATE INTERNAL STREAMS. # Internal Stream - out # It is used to output the top-k items and their counts after fixed intervals # Each element of the stream is a list of 2-sized tuples where, # tuple[1] = item # tuple[0] = its count # out = Stream() def count(counter, h, p, v): """ Given the counter and the hash functions, this function returns the count of the item Attributes ---------- counter: 2-d array Stores the hash bucket values h: 2-dimensional list of ints defines the hash function used to decide which bucket the count of v should be added to p: int the prime number used to create both the hash functions v: int An element of the stream Returns -------- co: int Count of the input item """ co = -1 for i in range(counter.shape[0]): v_bin = (v**2) * h[i][0] + v * h[i][1] + h[i][2] v_bin = v_bin % p % b temp = counter[i][v_bin] if co == -1 or temp < co: co = temp return int(co) #DEFINE THE COUNTER UPDATER #Updates the counter for every value in the stream def counter_update(v, state, h, p, k): """ This function updates the counter for a given value. If it sees the end of a finite stream it writes (pickles) the heap of top-k items into the file CMS.dat. Attributes ---------- v: int An element of the stream state: multidimensional list state stores the hash bucket (called counter), number of elements seen and the heap containing the top-k elements counter = state[0]: 2-d array number of elements = state[1]: int heap: = state[2]: array of 2-element tuples h: 2-dimensional list of ints defines the hash function used to decide which bucket the count of v should be added to p: int the prime number used to create both the hash functions k: int Specifies the number of heavy hitter elements """ counter = state[0] heap = state[2] if v == 'eof': heap = np.array(heap) np.savetxt("CMS.dat", heap) return state[2], state state[1] += 1 for i in range(counter.shape[0]): v_bin = (v**2) * h[i][0] + v * h[i][1] + h[i][2] v_bin = v_bin % p % b counter[i][v_bin] += 1 v_count = count(counter, h, p, v) if v_count >= state[1] / k: for i in range(len(heap)): if v == heap[i][1]: heap[i] = (v_count, v) heapq.heapify(heap) break else: heapq.heappush(heap, (v_count, v)) while len(heap) > k: heapq.heappop(heap) state[0] = counter state[2] = heap return state[2], state def regular_print(v): """ This function prints the last value of a window of the input stream """ print v[-1] # CREATE AGENTS # Make the agent that applies the counter update # function to the input stream map_element(func=counter_update, in_stream=in_streams[0], out_stream=out, state=state, h=h, p=p, k=k) #Make the agent which prints the counter stream at regular intervals sink_window(func=regular_print, in_stream=out, window_size=printer, step_size=printer)