def filter_stream(self, in_stream, out_stream): """ Filters the input stream to get the output stream using the function filter_element. """ map_element(self.filter_element, in_stream, out_stream)
def filter_stream(self, in_stream, out_stream): """ Filters the input stream to get the output stream using filter_sample(). """ map_element(self.filter_sample, in_stream, out_stream)
def heavy_hitters_stream(in_stream, out_stream, heavy_hitters_object): """ Parameters ---------- in_stream: Stream The input stream of the agent. An element of in_stream is the string version of a method call to a heavy_hitters object. For example 'add' for the method add, and 'heavy_hitters' for the method heavy_hitters. out_stream: Stream The output stream of the agent. Each element of the output stream is a dict which represents the heavy hitters. heavy_hitters_object: HeavyHitters An instance of HeavyHitters. """ def func(element): if element is 'heavy_hitters': return copy.copy(heavy_hitters_object.heavy_hitters) function_name, obj = element if function_name == 'add': heavy_hitters_object.add(obj) else: # The only functions supported here are heavy_hitters # and add. Include more functions when needed. raise ValueError return _no_value map_element(func, in_stream, out_stream)
def compute_func(in_streams, out_streams): eg = example_class(multiplicand=2) check_list = [0, 2, 5, 9, 14, 20, 27, 35, 44, 54] t = Stream() map_element(func=eg.step, in_stream=in_streams[0], out_stream=t) check_correctness_of_output(in_stream=t, check_list=check_list) stream_to_file(in_stream=t, filename='map_element_example_3.dat')
def f(in_streams, out_streams): def identity(v): return v map_element(func=identity, in_stream=in_streams[0], out_stream=out_streams[0])
def make_echo(spoken, D, A): """ Parameters ---------- spoken: Stream The stream of the spoken or created sound. D: number The Delay The delay from creation of sound to hearing the first echo. The units of D are the number of points in the sound input. A: floating point number The Attenuation factor. The ratio of the intensity of the spoken sound to its first echol. Notes ----- echo is the echo of the spoken sound heard is the sound that is heard; it is the spoken sound and its echo. """ echo = Stream(name='echo', initial_value=[0] * D) heard = spoken + echo map_element(func=lambda v: v * A, in_stream=heard, out_stream=echo) return heard
def compute(in_streams): def subtract_mean(window): return window[-1] - sum(window) / float(len(window)) def magnitude_of_vector(coordinates): return math.sqrt(sum([v * v for v in coordinates])) def simple_anomaly(value, threshold): if value > threshold: return 1.0 else: return 0.0 zero_mean_streams = [ Stream('zero mean e'), Stream('zero mean n'), Stream('zero mean z') ] magnitude_stream = Stream('magnitude') anomaly_stream = Stream('anomalies') filenames = ['zero_mean_e.txt', 'zero_mean_n.txt', 'zero_mean_z.txt'] for i in range(3): map_window(func=subtract_mean, in_stream=in_streams[i], out_stream=zero_mean_streams[i], window_size=8000, step_size=1) zip_map(func=magnitude_of_vector, in_streams=zero_mean_streams, out_stream=magnitude_stream) map_element(func=simple_anomaly, in_stream=magnitude_stream, out_stream=anomaly_stream, threshold=0.1) stream_to_file(in_stream=magnitude_stream, filename='magnitude.txt') stream_to_file(in_stream=anomaly_stream, filename='anomaly.txt')
def compute_func(in_streams, out_streams): import string f = string.upper check_list = map(f, source_list) t = Stream() map_element(func=f, in_stream=in_streams[0], out_stream=t) check_correctness_of_output(in_stream=t, check_list=check_list) stream_to_file(in_stream=t, filename='map_element_example_2.dat')
def compute_func(in_streams, out_streams): def f(x): return x * 10 check_list = map(f, source_list) t = Stream() map_element(func=f, in_stream=in_streams[0], out_stream=t) check_correctness_of_output(in_stream=t, check_list=check_list) stream_to_file(in_stream=t, filename='map_element_example_1.dat')
def compute_func_1(in_streams, out_streams): def get_sentiment(tweet): tweet_text = get_text(tweet) sentiment_of_tweet = sentiment_of_text(tweet_text) return (tweet_text, sentiment_of_tweet) map_element(func=get_sentiment, in_stream=in_streams[0], out_stream=out_streams[0])
def sieve(in_stream, prime_stream): print('in sieve. in_stream is ', in_stream) out_stream = Stream('y') print('in sieve. out_stream is ', out_stream) map_element(sieve_step, in_stream, out_stream, state=0, prime_stream=prime_stream)
def exponential_smooth_and_add(in_stream, out_stream, smoothing_factor): """ With input stream x and output stream y: y[n] = x[n] + a*x[n-1] + .. + a^m * x[n-m] + .... where a is smoothing_factor """ def f(in_stream_element, state, a): next_state = state * a + in_stream_element # return out_stream_element, next_state # Note: out_stream_element = next_state return next_state, next_state # Create a map_element agent map_element(f, in_stream, out_stream, state=0.0, a=smoothing_factor)
def compute_func(in_streams, out_streams): # This is a simple example of a composed agent consisting # of two component agents where the network has a single input # stream and no output stream. # The first component agent applies function f to each element # of in_stream, and puts the result in its output stream t. # The second component agent puts values in its input stream t # on a file called test.dat. # test.dat will contain 10, 20, 30, .... def f(x): return x * 10 t = Stream() map_element(func=f, in_stream=in_streams[0], out_stream=t) stream_to_file(in_stream=t, filename='test.dat')
def exponential_smoothing(in_stream, out_stream, smoothing_factor): """ With input stream x and output stream y: y[0] = x[0] y[n] = a*x[n] + (1-a)*y[n-1] for n > 0. where a is smoothing_factor """ def f(in_stream_element, state, a): if state is 'empty': next_state = in_stream_element else: next_state = (1.0 - a) * state + a * in_stream_element # return out_stream_element, next_state # Note: out_stream_element = next_state # So, return next_state, next_state return next_state, next_state # Create a map_element agent map_element(f, in_stream, out_stream, state='empty', a=smoothing_factor)
def g_function(): from op import map_element t = Stream('t') u = Stream('u') t1 = Stream('t1') def g_print(y): return y*2 def gg_print(y): print 'In g_function. gg_print() y is', y return 100*y map_element( func=g_print, in_stream=t, out_stream=t1, name='b') map_element( func=gg_print, in_stream=t1, out_stream=u, name='b1') sources = [] in_streams = [t] out_streams = [u] name_to_stream = {s.name: s for s in in_streams} Stream.scheduler.name_to_stream = name_to_stream return sources, in_streams, out_streams
def f_function(): from source import source_function from op import map_element s = Stream('s') t = Stream('t') def ff(x): return x*10 def gg(state): return state+1, state+1 map_element( func=ff, in_stream=s, out_stream=t, name='aaaa') ss = source_function( func=gg, stream_name='s', time_interval=0.1, num_steps=10, state=0, window_size=1, name='source') sources = [ss] in_streams = [s] out_streams = [t] name_to_stream = {s.name: s for s in in_streams} Stream.scheduler.name_to_stream = name_to_stream return sources, in_streams, out_streams
def bloom_filter_on_stream(in_stream, out_stream, blm): """ Parameters ---------- in_stream: Stream The input stream of the agent. Each element of the input stream is a pair: (function_name, string) where function_name is a string which is one of 'add', 'check', 'remove' out_stream: Stream The output stream of the agent. An element is added to the output stream when a 'check' function_name appears on the input stream. The output stream consists of pairs (string, boolean) where boolean is True if and only if string is in the input stream at this point. blm: BloomFilter An instance of BloomFilter. """ def func(element): function_name, obj = element if function_name == 'add': blm.add(obj) return _no_value elif function_name == 'remove': blm.remove(obj) return _no_value elif function_name == 'check': obj_is_in_input_stream = blm.check(obj) # True if obj is in the input stream at this point. # False otherwise. return (obj, obj_is_in_input_stream) else: raise ValueError map_element(func, in_stream, out_stream)
def misra_gries(k, in_stream, out_stream, M=1): """ This function creates an agent which executes the misra-gries heavy hitters algorithm on the input in_stream to produce the output stream, out_stream. Parameters ---------- k: int, positive Specifies the number of heavy hitter elements that the algorithm searches for. in_stream: Stream The input stream out_stream: Stream The output stream M: int, positive Outputs candidates for heavy hitters after every M new arrivals in in_stream. """ # CREATE AGENT # Make the agent that reads the input stream and # produces the output stream. # Set up the initial state. keys = [None] * k counts = [0] * k index = 0 initial_state = (keys, counts, index) # Create agent map_element(func=misra_gries_process_element, in_stream=in_stream, out_stream=out_stream, state=initial_state, M=M)
def datastore_file_manager(in_streams, out_streams): """ Agent that manages writing to and creating datastore files @in_streams - Stream of (n, e, z, t) @out_streams - Stream of (n, e, z, t) - stream itself is not modified """ def write_to_file(stream, state): """ Writes stream to file. Creates a new file after FILE_STORE_INTERVAL passes. @param stream - input stream of acceleration data. @param state - tuple of (timestamp, curr_file_pointer, n_writing_error) """ timestamp = stream[3] latest_timestamp, curr_file_ptr, curr_filename, n_writing_error = state # update the datastore file if FILE_STORE_INTERVAL passed if timestamp >= latest_timestamp + FILE_STORE_INTERVAL: if curr_file_ptr is not None: curr_file_ptr.close() curr_filename = make_filename(timestamp) curr_file_ptr = open(curr_filename, 'w') latest_timestamp = timestamp try: write_to_datastore_file(stream, curr_file_ptr) except: n_writing_error += 1 if n_writing_error <= 10: print( 'Error writing sample to file {} with timestamp {}'.format( curr_filename, timestamp)) return stream, (latest_timestamp, curr_file_ptr, curr_filename, n_writing_error) map_element(write_to_file, in_streams, out_streams, state=(-FILE_STORE_INTERVAL, None, None, 0))
def membership_in_stream(in_stream, out_stream, membership_object): """ Parameters ---------- in_stream: Stream The input stream of the agent. Each element of the input stream is a pair: (function_name, string) where function_name is a string which is one of 'add', 'check', 'remove' or other string associated with a method of membership_object. out_stream: Stream The output stream of the agent. The output stream contains results of executing the method specified by function name on membership_object. membership_object: Membership An instance of a Membership class such as BloomFilter or CountMinSketch from PyProbables. """ def func(element): # Each element of the input stream is assumed to be a # pair: function_name and a value. function_name, value = element if function_name == 'add': membership_object.add(value) return _no_value elif function_name == 'remove': membership_object.remove(value) return _no_value elif function_name == 'check': return (value, membership_object.check(value)) else: raise ValueError map_element(func, in_stream, out_stream)
def run_Paxos(): proposers = [ blend(func=proposer_behavior, in_streams=daemons_proposers_out_streams, out_stream=proposer_out_streams[id], state=(id, 0, random_number(), {}), name='proposer_' + str(id)) for id in range(num_proposers) ] acceptors = [ map_element(func=acceptor_behavior, in_stream=daemons_acceptors_out_streams[id], out_stream=acceptor_out_streams[id], state=(id, -1, -1, -1, -1), name='acceptor_' + str(id)) for id in range(num_acceptors) ] time_step_agent = blend(func=time_step_behavior, in_streams=in_streams_for_time_step, out_stream=time_step_stream, state=({}, None), name='time step agent') daemons_acceptors = [ blend(func=delete, in_streams=proposer_out_streams, out_stream=daemons_acceptors_out_streams[id], name='daemons_acceptors_' + str(id)) for id in range(num_acceptors) ] daemons_proposers = [ blend(func=delete, in_streams=acceptor_out_streams, out_stream=daemons_proposers_out_streams[id], name='daemons_proposers_' + str(id)) for id in range(num_proposers) ] #STEP 4. START COMPUTATION # Get the scheduler and execute a step. scheduler = Stream.scheduler # Start the computation by putting a value into the # time step stream time_step_stream.append(('time_step', 1)) # Start the scheduler. scheduler.step()
def fprepend(lst, in_stream): out_stream = Stream() out_stream.extend(lst) map_element(lambda v: v, in_stream, out_stream) return out_stream
def prepend(lst, in_stream, out_stream): out_stream.extend(lst) map_element(lambda v: v, in_stream, out_stream)
def g(in_stream, out_stream, **kwargs): map_element(func, in_stream, out_stream, **kwargs) return out_stream
def f(in_streams, out_streams): map_element(func=lambda x: 7 * x, in_stream=in_streams[0], out_stream=out_streams[0])
def compute(in_streams, out_streams): map_element(func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0])
def compute_1(in_streams, out_streams): result_stream = Stream('result of computation') map_element(func=lambda x: 200 * x, in_stream=in_streams[0], out_stream=result_stream) stream_to_file(in_stream=result_stream, filename='result.dat')
def primes_example_2(N): """ Agent used in example 2 in which prime_stream is the sequence of primes up to the N-th prime Parameters ---------- N: int positive integer Returns: first_N, prime_stream ------- first_N: list The first N primes prime_stream: Stream Stream of prime numbers. May have more than N primes Notes ----- sieve creates a single sink agent. The sink agent has a single input stream, in_stream. The agent encapsulates stateful function f which has an initial state of 0. (Sinks have no output streams.) Let the first element of in_stream be p. This agent assumes that p is a prime number. So, the agent appends p to prime_stream. Many agents append prime numbers to prime_stream, but at most one agent can do so at a time. When the agent discovers an element of in_stream that is not a multiple of p, the agent creates a new sieve agent which takes a new stream out_stream as its input stream. out_stream consists of elements of in_stream that are not multiples of p. """ def execute_until_stop_message(v, state, function): function_state, finished_execution = state if finished_execution: return (_no_value, True) index, input_value = v if index == 1: # This value is from stop_stream # Make finished_execution become True because a message # was received on stop_stream. finished_execution = True # From now onwards, no messages are appended to the output # stream, and finished_execution remains True forever. return (_no_value, (function_state, True)) # index is 0. So, this value is from state_stream. output_value, next_function_state = function(input_value, function_state) # next_state = (next_function_state, finished_execution) return output_value, (next_function_state, finished_execution) def generate_numbers_until_stop_message(index_and_value, state): # state is initially False and switches to True if a message # is received in stop_stream. If state becomes True then it # remains True thereafter. After state becomes True no values # are appended to the output stream. # The elements of the input stream are tuples: index and # value. # index is 0 for state_stream and 1 for stop_stream. index, value = index_and_value if index == 1: # This value is from stop_stream # Make state True because a message was received on # stop_stream. # From now onwards, no messages are appended to the output # stream, and state remains True. return (_no_value, True) # index is 0. So, this value is from state_stream. if state: # Do not append values to the output stream, and state # remains True return (_no_value, state) else: # Append the next value to the output stream, and state # remains False. return (value + 1, state) def detect_finished_then_send_stop(v, state, N): length, stop = state # If stop is True then computation must stop length += 1 if length >= N and not stop: stop = True return (True, (length, stop)) else: return (_no_value, (length, stop)) def first_N_elements(in_stream, N, first_N): def first_N_elements_of_stream(v, state, N, first_N): if state < N: first_N.append(v) state += 1 return state sink(func=first_N_elements_of_stream, in_stream=in_stream, state=0, N=N, first_N=first_N) #----------------------------------------------------------------- # Define streams #----------------------------------------------------------------- state_stream = Stream(name='numbers 2, 3, 4, ...') stop_stream = Stream(name='stop!') prime_stream = Stream(name='prime numbers') first_N = [] #----------------------------------------------------------------- # Define agents #----------------------------------------------------------------- # Create agent that generates 2, 3, 4... until it receives a # message on stop_stream ## merge_asynch(func=generate_numbers_until_stop_message, ## in_streams=[state_stream, stop_stream], ## out_stream=state_stream, state=False) def g(v, state): return v + 1, state merge_asynch(func=execute_until_stop_message, in_streams=[state_stream, stop_stream], out_stream=state_stream, state=(None, False), function=g) # Create an agent that sieves state_stream to create prime_stream # which is a sequence of primes. # We do this by creating a sink agent that encapsulates a stateful # function f with an initial state of 0. Pass parameters # prime_stream and out_stream from the sink agent to its # encapsulated function f. sieve(in_stream=state_stream, prime_stream=prime_stream) # Create an agent that sends a message on stop_stream when the # length of prime_stream exceeds N. map_element(func=detect_finished_then_send_stop, in_stream=prime_stream, out_stream=stop_stream, state=(0, False), N=N) first_N_elements(in_stream=prime_stream, N=N, first_N=first_N) state_stream.append(2) return first_N, prime_stream
def compute_func_2(in_streams, out_streams): map_element(func=followers_and_retweets_of_tweet, in_stream=in_streams[0], out_stream=out_streams[0])
def compute_func_0(in_streams, out_streams): map_element(lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0])