def filter_stream(self, in_stream, out_stream):
"""
Filters the input stream to get the output stream using
the function filter_element.
"""
map_element(self.filter_element, in_stream, out_stream)
def filter_stream(self, in_stream, out_stream):
"""
Filters the input stream to get the output stream
using filter_sample().
"""
map_element(self.filter_sample, in_stream, out_stream)
def heavy_hitters_stream(in_stream, out_stream, heavy_hitters_object):
"""
Parameters
----------
in_stream: Stream
The input stream of the agent.
An element of in_stream is the string version of a
method call to a heavy_hitters object. For example
'add' for the method add, and 'heavy_hitters' for
the method heavy_hitters.
out_stream: Stream
The output stream of the agent.
Each element of the output stream is a dict which
represents the heavy hitters.
heavy_hitters_object: HeavyHitters
An instance of HeavyHitters.
"""
def func(element):
if element is 'heavy_hitters':
return copy.copy(heavy_hitters_object.heavy_hitters)
function_name, obj = element
if function_name == 'add':
heavy_hitters_object.add(obj)
else:
# The only functions supported here are heavy_hitters
# and add. Include more functions when needed.
raise ValueError
return _no_value
map_element(func, in_stream, out_stream)
def compute_func(in_streams, out_streams):
eg = example_class(multiplicand=2)
check_list = [0, 2, 5, 9, 14, 20, 27, 35, 44, 54]
t = Stream()
map_element(func=eg.step, in_stream=in_streams[0], out_stream=t)
check_correctness_of_output(in_stream=t, check_list=check_list)
stream_to_file(in_stream=t, filename='map_element_example_3.dat')
def f(in_streams, out_streams):
def identity(v):
return v
map_element(func=identity,
in_stream=in_streams[0],
out_stream=out_streams[0])
def make_echo(spoken, D, A):
"""
Parameters
----------
spoken: Stream
The stream of the spoken or created sound.
D: number
The Delay
The delay from creation of sound to hearing the
first echo. The units of D are the number of
points in the sound input.
A: floating point number
The Attenuation factor.
The ratio of the intensity of the spoken sound
to its first echol.
Notes
-----
echo is the echo of the spoken sound
heard is the sound that is heard; it is the spoken
sound and its echo.
"""
echo = Stream(name='echo', initial_value=[0] * D)
heard = spoken + echo
map_element(func=lambda v: v * A, in_stream=heard, out_stream=echo)
return heard
def compute(in_streams):
def subtract_mean(window):
return window[-1] - sum(window) / float(len(window))
def magnitude_of_vector(coordinates):
return math.sqrt(sum([v * v for v in coordinates]))
def simple_anomaly(value, threshold):
if value > threshold: return 1.0
else: return 0.0
zero_mean_streams = [
Stream('zero mean e'),
Stream('zero mean n'),
Stream('zero mean z')
]
magnitude_stream = Stream('magnitude')
anomaly_stream = Stream('anomalies')
filenames = ['zero_mean_e.txt', 'zero_mean_n.txt', 'zero_mean_z.txt']
for i in range(3):
map_window(func=subtract_mean,
in_stream=in_streams[i],
out_stream=zero_mean_streams[i],
window_size=8000,
step_size=1)
zip_map(func=magnitude_of_vector,
in_streams=zero_mean_streams,
out_stream=magnitude_stream)
map_element(func=simple_anomaly,
in_stream=magnitude_stream,
out_stream=anomaly_stream,
threshold=0.1)
stream_to_file(in_stream=magnitude_stream, filename='magnitude.txt')
stream_to_file(in_stream=anomaly_stream, filename='anomaly.txt')
def compute_func(in_streams, out_streams):
import string
f = string.upper
check_list = map(f, source_list)
t = Stream()
map_element(func=f, in_stream=in_streams[0], out_stream=t)
check_correctness_of_output(in_stream=t, check_list=check_list)
stream_to_file(in_stream=t, filename='map_element_example_2.dat')
def compute_func(in_streams, out_streams):
def f(x):
return x * 10
check_list = map(f, source_list)
t = Stream()
map_element(func=f, in_stream=in_streams[0], out_stream=t)
check_correctness_of_output(in_stream=t, check_list=check_list)
stream_to_file(in_stream=t, filename='map_element_example_1.dat')
def compute_func_1(in_streams, out_streams):
def get_sentiment(tweet):
tweet_text = get_text(tweet)
sentiment_of_tweet = sentiment_of_text(tweet_text)
return (tweet_text, sentiment_of_tweet)
map_element(func=get_sentiment,
in_stream=in_streams[0],
out_stream=out_streams[0])
def sieve(in_stream, prime_stream):
print('in sieve. in_stream is ', in_stream)
out_stream = Stream('y')
print('in sieve. out_stream is ', out_stream)
map_element(sieve_step,
in_stream,
out_stream,
state=0,
prime_stream=prime_stream)
def exponential_smooth_and_add(in_stream, out_stream, smoothing_factor):
"""
With input stream x and output stream y:
y[n] = x[n] + a*x[n-1] + .. + a^m * x[n-m] + ....
where a is smoothing_factor
"""
def f(in_stream_element, state, a):
next_state = state * a + in_stream_element
# return out_stream_element, next_state
# Note: out_stream_element = next_state
return next_state, next_state
# Create a map_element agent
map_element(f, in_stream, out_stream, state=0.0, a=smoothing_factor)
def compute_func(in_streams, out_streams):
# This is a simple example of a composed agent consisting
# of two component agents where the network has a single input
# stream and no output stream.
# The first component agent applies function f to each element
# of in_stream, and puts the result in its output stream t.
# The second component agent puts values in its input stream t
# on a file called test.dat.
# test.dat will contain 10, 20, 30, ....
def f(x):
return x * 10
t = Stream()
map_element(func=f, in_stream=in_streams[0], out_stream=t)
stream_to_file(in_stream=t, filename='test.dat')
def exponential_smoothing(in_stream, out_stream, smoothing_factor):
"""
With input stream x and output stream y:
y[0] = x[0]
y[n] = a*x[n] + (1-a)*y[n-1] for n > 0.
where a is smoothing_factor
"""
def f(in_stream_element, state, a):
if state is 'empty':
next_state = in_stream_element
else:
next_state = (1.0 - a) * state + a * in_stream_element
# return out_stream_element, next_state
# Note: out_stream_element = next_state
# So, return next_state, next_state
return next_state, next_state
# Create a map_element agent
map_element(f, in_stream, out_stream, state='empty', a=smoothing_factor)
def g_function():
from op import map_element
t = Stream('t')
u = Stream('u')
t1 = Stream('t1')
def g_print(y):
return y*2
def gg_print(y):
print 'In g_function. gg_print() y is', y
return 100*y
map_element(
func=g_print, in_stream=t, out_stream=t1, name='b')
map_element(
func=gg_print, in_stream=t1, out_stream=u, name='b1')
sources = []
in_streams = [t]
out_streams = [u]
name_to_stream = {s.name: s for s in in_streams}
Stream.scheduler.name_to_stream = name_to_stream
return sources, in_streams, out_streams
def f_function():
from source import source_function
from op import map_element
s = Stream('s')
t = Stream('t')
def ff(x):
return x*10
def gg(state):
return state+1, state+1
map_element(
func=ff, in_stream=s, out_stream=t, name='aaaa')
ss = source_function(
func=gg, stream_name='s',
time_interval=0.1, num_steps=10, state=0, window_size=1,
name='source')
sources = [ss]
in_streams = [s]
out_streams = [t]
name_to_stream = {s.name: s for s in in_streams}
Stream.scheduler.name_to_stream = name_to_stream
return sources, in_streams, out_streams
def bloom_filter_on_stream(in_stream, out_stream, blm):
"""
Parameters
----------
in_stream: Stream
The input stream of the agent.
Each element of the input stream is a pair:
(function_name, string) where function_name is
a string which is one of 'add', 'check', 'remove'
out_stream: Stream
The output stream of the agent. An element is
added to the output stream when a 'check'
function_name appears on the input stream.
The output stream consists of pairs (string, boolean)
where boolean is True if and only if string
is in the input stream at this point.
blm: BloomFilter
An instance of BloomFilter.
"""
def func(element):
function_name, obj = element
if function_name == 'add':
blm.add(obj)
return _no_value
elif function_name == 'remove':
blm.remove(obj)
return _no_value
elif function_name == 'check':
obj_is_in_input_stream = blm.check(obj)
# True if obj is in the input stream at this point.
# False otherwise.
return (obj, obj_is_in_input_stream)
else:
raise ValueError
map_element(func, in_stream, out_stream)
def misra_gries(k, in_stream, out_stream, M=1):
"""
This function creates an agent which
executes the misra-gries heavy hitters algorithm
on the input in_stream to produce the output
stream, out_stream.
Parameters
----------
k: int, positive
Specifies the number of heavy hitter elements
that the algorithm searches for.
in_stream: Stream
The input stream
out_stream: Stream
The output stream
M: int, positive
Outputs candidates for heavy hitters after every
M new arrivals in in_stream.
"""
# CREATE AGENT
# Make the agent that reads the input stream and
# produces the output stream.
# Set up the initial state.
keys = [None] * k
counts = [0] * k
index = 0
initial_state = (keys, counts, index)
# Create agent
map_element(func=misra_gries_process_element,
in_stream=in_stream,
out_stream=out_stream,
state=initial_state,
M=M)
def datastore_file_manager(in_streams, out_streams):
""" Agent that manages writing to and creating datastore files
@in_streams - Stream of (n, e, z, t)
@out_streams - Stream of (n, e, z, t) - stream itself is not modified
"""
def write_to_file(stream, state):
""" Writes stream to file. Creates a new file after FILE_STORE_INTERVAL passes.
@param stream - input stream of acceleration data.
@param state - tuple of (timestamp, curr_file_pointer, n_writing_error)
"""
timestamp = stream[3]
latest_timestamp, curr_file_ptr, curr_filename, n_writing_error = state
# update the datastore file if FILE_STORE_INTERVAL passed
if timestamp >= latest_timestamp + FILE_STORE_INTERVAL:
if curr_file_ptr is not None:
curr_file_ptr.close()
curr_filename = make_filename(timestamp)
curr_file_ptr = open(curr_filename, 'w')
latest_timestamp = timestamp
try:
write_to_datastore_file(stream, curr_file_ptr)
except:
n_writing_error += 1
if n_writing_error <= 10:
print(
'Error writing sample to file {} with timestamp {}'.format(
curr_filename, timestamp))
return stream, (latest_timestamp, curr_file_ptr, curr_filename,
n_writing_error)
map_element(write_to_file,
in_streams,
out_streams,
state=(-FILE_STORE_INTERVAL, None, None, 0))
def membership_in_stream(in_stream, out_stream, membership_object):
"""
Parameters
----------
in_stream: Stream
The input stream of the agent.
Each element of the input stream is a pair:
(function_name, string) where function_name is
a string which is one of 'add', 'check', 'remove'
or other string associated with a method of
membership_object.
out_stream: Stream
The output stream of the agent. The output stream
contains results of executing the method specified
by function name on membership_object.
membership_object: Membership
An instance of a Membership class such as BloomFilter
or CountMinSketch from PyProbables.
"""
def func(element):
# Each element of the input stream is assumed to be a
# pair: function_name and a value.
function_name, value = element
if function_name == 'add':
membership_object.add(value)
return _no_value
elif function_name == 'remove':
membership_object.remove(value)
return _no_value
elif function_name == 'check':
return (value, membership_object.check(value))
else:
raise ValueError
map_element(func, in_stream, out_stream)
def run_Paxos():
proposers = [
blend(func=proposer_behavior,
in_streams=daemons_proposers_out_streams,
out_stream=proposer_out_streams[id],
state=(id, 0, random_number(), {}),
name='proposer_' + str(id)) for id in range(num_proposers)
]
acceptors = [
map_element(func=acceptor_behavior,
in_stream=daemons_acceptors_out_streams[id],
out_stream=acceptor_out_streams[id],
state=(id, -1, -1, -1, -1),
name='acceptor_' + str(id)) for id in range(num_acceptors)
]
time_step_agent = blend(func=time_step_behavior,
in_streams=in_streams_for_time_step,
out_stream=time_step_stream,
state=({}, None),
name='time step agent')
daemons_acceptors = [
blend(func=delete,
in_streams=proposer_out_streams,
out_stream=daemons_acceptors_out_streams[id],
name='daemons_acceptors_' + str(id))
for id in range(num_acceptors)
]
daemons_proposers = [
blend(func=delete,
in_streams=acceptor_out_streams,
out_stream=daemons_proposers_out_streams[id],
name='daemons_proposers_' + str(id))
for id in range(num_proposers)
]
#STEP 4. START COMPUTATION
# Get the scheduler and execute a step.
scheduler = Stream.scheduler
# Start the computation by putting a value into the
# time step stream
time_step_stream.append(('time_step', 1)) # Start the scheduler.
scheduler.step()
def fprepend(lst, in_stream):
out_stream = Stream()
out_stream.extend(lst)
map_element(lambda v: v, in_stream, out_stream)
return out_stream
def prepend(lst, in_stream, out_stream):
out_stream.extend(lst)
map_element(lambda v: v, in_stream, out_stream)
def g(in_stream, out_stream, **kwargs):
map_element(func, in_stream, out_stream, **kwargs)
return out_stream
def f(in_streams, out_streams):
map_element(func=lambda x: 7 * x,
in_stream=in_streams[0],
out_stream=out_streams[0])
def compute(in_streams, out_streams):
map_element(func=lambda x: x,
in_stream=in_streams[0],
out_stream=out_streams[0])
def compute_1(in_streams, out_streams):
result_stream = Stream('result of computation')
map_element(func=lambda x: 200 * x,
in_stream=in_streams[0],
out_stream=result_stream)
stream_to_file(in_stream=result_stream, filename='result.dat')
def primes_example_2(N):
"""
Agent used in example 2 in which prime_stream is the sequence of
primes up to the N-th prime
Parameters
----------
N: int
positive integer
Returns: first_N, prime_stream
-------
first_N: list
The first N primes
prime_stream: Stream
Stream of prime numbers. May have more than N primes
Notes
-----
sieve creates a single sink agent. The sink agent has a single
input stream, in_stream. The agent encapsulates stateful function
f which has an initial state of 0. (Sinks have no output streams.)
Let the first element of in_stream be p. This agent assumes that p
is a prime number. So, the agent appends p to prime_stream. Many
agents append prime numbers to prime_stream, but at most one agent
can do so at a time.
When the agent discovers an element of in_stream that is not a
multiple of p, the agent creates a new sieve agent which takes a
new stream out_stream as its input stream. out_stream consists of
elements of in_stream that are not multiples of p.
"""
def execute_until_stop_message(v, state, function):
function_state, finished_execution = state
if finished_execution:
return (_no_value, True)
index, input_value = v
if index == 1:
# This value is from stop_stream
# Make finished_execution become True because a message
# was received on stop_stream.
finished_execution = True
# From now onwards, no messages are appended to the output
# stream, and finished_execution remains True forever.
return (_no_value, (function_state, True))
# index is 0. So, this value is from state_stream.
output_value, next_function_state = function(input_value,
function_state)
# next_state = (next_function_state, finished_execution)
return output_value, (next_function_state, finished_execution)
def generate_numbers_until_stop_message(index_and_value, state):
# state is initially False and switches to True if a message
# is received in stop_stream. If state becomes True then it
# remains True thereafter. After state becomes True no values
# are appended to the output stream.
# The elements of the input stream are tuples: index and
# value.
# index is 0 for state_stream and 1 for stop_stream.
index, value = index_and_value
if index == 1:
# This value is from stop_stream
# Make state True because a message was received on
# stop_stream.
# From now onwards, no messages are appended to the output
# stream, and state remains True.
return (_no_value, True)
# index is 0. So, this value is from state_stream.
if state:
# Do not append values to the output stream, and state
# remains True
return (_no_value, state)
else:
# Append the next value to the output stream, and state
# remains False.
return (value + 1, state)
def detect_finished_then_send_stop(v, state, N):
length, stop = state
# If stop is True then computation must stop
length += 1
if length >= N and not stop:
stop = True
return (True, (length, stop))
else:
return (_no_value, (length, stop))
def first_N_elements(in_stream, N, first_N):
def first_N_elements_of_stream(v, state, N, first_N):
if state < N:
first_N.append(v)
state += 1
return state
sink(func=first_N_elements_of_stream,
in_stream=in_stream,
state=0,
N=N,
first_N=first_N)
#-----------------------------------------------------------------
# Define streams
#-----------------------------------------------------------------
state_stream = Stream(name='numbers 2, 3, 4, ...')
stop_stream = Stream(name='stop!')
prime_stream = Stream(name='prime numbers')
first_N = []
#-----------------------------------------------------------------
# Define agents
#-----------------------------------------------------------------
# Create agent that generates 2, 3, 4... until it receives a
# message on stop_stream
## merge_asynch(func=generate_numbers_until_stop_message,
## in_streams=[state_stream, stop_stream],
## out_stream=state_stream, state=False)
def g(v, state):
return v + 1, state
merge_asynch(func=execute_until_stop_message,
in_streams=[state_stream, stop_stream],
out_stream=state_stream,
state=(None, False),
function=g)
# Create an agent that sieves state_stream to create prime_stream
# which is a sequence of primes.
# We do this by creating a sink agent that encapsulates a stateful
# function f with an initial state of 0. Pass parameters
# prime_stream and out_stream from the sink agent to its
# encapsulated function f.
sieve(in_stream=state_stream, prime_stream=prime_stream)
# Create an agent that sends a message on stop_stream when the
# length of prime_stream exceeds N.
map_element(func=detect_finished_then_send_stop,
in_stream=prime_stream,
out_stream=stop_stream,
state=(0, False),
N=N)
first_N_elements(in_stream=prime_stream, N=N, first_N=first_N)
state_stream.append(2)
return first_N, prime_stream
def compute_func_2(in_streams, out_streams):
map_element(func=followers_and_retweets_of_tweet,
in_stream=in_streams[0],
out_stream=out_streams[0])
def compute_func_0(in_streams, out_streams):
map_element(lambda x: x,
in_stream=in_streams[0],
out_stream=out_streams[0])
