def two_process_publisher(): source_list = range(10) def source(out_stream): return source_list_to_stream(source_list, out_stream) def compute_0(in_streams, out_streams): map_element( func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) proc_0 = distributed_process( compute_func=compute_0, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', source)], name='process_0') def compute_1(in_streams, out_streams): map_element( func=lambda x: x+10, in_stream=in_streams[0], out_stream=out_streams[0]) proc_1 = distributed_process( compute_func=compute_1, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[], name='process_1' ) vm_0 = VM( processes=[proc_0, proc_1], connections=[(proc_0, 'out', proc_1, 'in')], publishers=[(proc_1, 'out', 'publication')]) vm_0.start()
def single_process_subscriber(): # This VM has a single process called proc_1. # This process has a single input stream that we # call 'in' and it has no output streams; so # out_stream_names is empty. Elements arriving on # the input stream are copied to a file called # result.dat (See compute_func.) # This process has no source threads that # generate data. The input comes only from # subscribing to a stream. Because this process # has no source threads, connect_sources is # empty. Likewise, since it has no actuators, # connect_actuators is empty. def compute_func(in_streams, out_streams): stream_to_file(in_streams[0], 'result.dat') proc_1 = distributed_process(compute_func=compute_func, in_stream_names=['in'], out_stream_names=[], connect_sources=[], connect_actuators=[], name='proc_1') # This VM consists of a single process. So, it has # no connections to other processes within the same # shared-memory multicore machine. # It is a subscriber to a stream called # copy_of_source_list # Elements received on this stream are passed to the # stream called 'in' inside the process called proc_1. vm_1 = VM(processes=[proc_1], connections=[], subscribers=[(proc_1, 'in', 'copy_of_source_list')]) vm_1.start()
def global_aggregator(): def compute_func(in_streams, out_streams): """ Parameters ---------- in_streams: list of Stream in_streams is a list of anomaly streams with one stream from each sensor. An anomaly stream is a sequence of 0.0 and 1.0 where 0.0 indicates no anomaly and 1.0 indicates an anomaly. out_streams: list of Stream This list consists of a single stream that contains 0.0 when no global anomaly across all sensors is detected and 1.0 when a global anomaly is detected. """ aggregate_anomalies(in_streams, out_streams, timed_window_size=2) proc = distributed_process(compute_func=compute_func, in_stream_names=['in_1', 'in_2'], out_stream_names=[], connect_sources=[], name='global aggregator') vm = VM(processes=[proc], connections=[], subscribers=[(proc, 'in_1', 'S1'), (proc, 'in_2', 'S2')]) vm.start()
def single_process_publication_subscriber(): """ The application in this example consists of single process. The process has no source and no actuator. It has a single in_stream called 'in'. This example creates a virtual machine (vm) which subscribes to a stream called 'sequence'. The steps for creating a process are: (1) Define the sources. In this example we have no sources. (2) Define the actuators. In this example we have no actuators. (3) Define compute_func. This process has a single input stream and no output stream. (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the virtual machine.. """ # SKIP STEPS 1, 2 BECAUSE NO SOURCES OR ACTUATORS. # STEP 3: DEFINE COMPUTE_FUNC def g(in_streams, out_streams): def print_element(v): print 'stream element is ', v sink_element(func=print_element, in_stream=in_streams[0]) # STEP 4: CREATE PROCESSES proc_1 = distributed_process(compute_func=g, in_stream_names=['in'], out_stream_names=[], connect_sources=[], connect_actuators=[], name='proc_1') # FINAL STEP: CREATE A VM AND START IT. # Since this application has a single process it has no # connections between processes. The process, proc_1, subscribes # to a stream called 'sequence'. This process does not publish # streams. vm_1 = VM(processes=[proc_1], connections=[], subscribers=[(proc_1, 'in', 'sequence')]) vm_1.start()
def two_process_subscriber(): def compute_2(in_streams, out_streams): map_element(func=lambda x: x * 100, in_stream=in_streams[0], out_stream=out_streams[0]) proc_2 = distributed_process(compute_func=compute_2, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[], name='process_3') def compute_3(in_streams, out_streams): stream_to_file(in_streams[0], 'result.dat') proc_3 = distributed_process(compute_func=compute_3, in_stream_names=['in'], out_stream_names=[], connect_sources=[], name='process_1') vm_1 = VM(processes=[proc_2, proc_3], connections=[(proc_2, 'out', proc_3, 'in')], subscribers=[(proc_2, 'in', 'publication')]) vm_1.start()
def single_process_publisher(): # This VM has a single process called proc_0. # This process has a single input stream called # 'in' and a single output stream called 'out'. See # in_stream_names=['in'], out_stream_names=['out']. # It has a single source thread which puts elements # of source_list into the stream called 'in'. # The process has no actuators, and so connect_actuators # is empty. See connect_sources=[('in', source)], # connect_actuators=[]. # The computational thread of this process merely # copies its input stream to its output stream (see # compute_func.) # The process publishes its output stream with the # publication name 'copy_of_source_list'. See # publishers=[(proc_0, 'out', 'copy_of_source_list')] # This VM consists of a single process and so it has # no connections to other processes within the same # multicore machine; so, connections is the empty list. source_list = range(10) def source(out_stream): return source_list_to_stream( source_list, out_stream, time_interval=0.01) def compute_func(in_streams, out_streams): map_element( func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) proc_0 = distributed_process( compute_func=compute_func, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', source)], connect_actuators=[], name='proc_0') vm_0 = VM( processes=[proc_0], connections=[], publishers=[(proc_0, 'out', 'copy_of_source_list')]) vm_0.start()
def single_process_publisher(): source_list = range(10) def source(out_stream): return source_list_to_stream(source_list, out_stream, time_interval=0.2) def compute_func(in_streams, out_streams): map_element(func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) proc_0 = distributed_process(compute_func=compute_func, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', source)], connect_actuators=[], name='proc_0') vm_0 = VM(processes=[proc_0], connections=[], publishers=[(proc_0, 'out', 'copy_of_source_list')]) vm_0.start()
def single_process_publication_producer(): """ The application in this example consists of single process. The process has a single source and no actuator. The single source generates 1, 2, 3, 4, ..... The compute function multiplies this sequence by 10 and puts the result in the file called test.dat num_steps is the number of values output by the source. For example, if num_steps is 4 and test.dat is empty before the function is called then, test.dat will contain 10, 20, 30, 40 on separate lines. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the application. """ # STEP 1: DEFINE SOURCES def source(out_stream): """ A simple source which outputs 1, 2, 3,... on out_stream. """ def generate_sequence(state): return state + 1, state + 1 # Return an agent which takes 10 steps, and # sleeps for 0.1 seconds between successive steps, and # puts the next element of the sequence in stream s, # and starts the sequence with value 0. The elements on # out_stream will be 1, 2, 3, ... return source_func_to_stream(func=generate_sequence, out_stream=out_stream, time_interval=0.1, num_steps=10, state=0) # STEP 2: DEFINE ACTUATORS # This example has no actuators # STEP 3: DEFINE COMPUTE_FUNC def f(in_streams, out_streams): map_element(func=lambda x: 7 * x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES # This process has a single input stream that we call 'in' and it # has no output streams. We connect the source to the input stream # called 'in'. proc_0 = distributed_process(compute_func=f, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', source)], connect_actuators=[], name='proc_0') # FINAL STEP: CREATE A VM AND START IT. # Since this application has a single process it has no # connections between processes. The process, proc_0, publishes # its output stream called 'out' to a stream called # 'sequence'. This process does not subscribe to streams. vm_0 = VM(processes=[proc_0], connections=[], publishers=[(proc_0, 'out', 'sequence')]) vm_0.start()
def clock_offset_estimation_multiprocess(): """ An example of a multiprocess app. This example has three processes: proc_0 and proc_1 get time offsets from an ntp server, and put them on output streams. proc_2 gets these two streams as input, merges them and puts the resulting stream on a file called 'offsets.dat'. """ # ---------------------------------------------------------------- # DEFINE EACH OF THE PROCESSES # ---------------------------------------------------------------- # The steps for creating a process are: # STEP 1: Define the sources: source() # STEP 2: Define the computational network: compute() # STEP 3: Call single_process_multiple_sources() # Carry out the above three steps for each process # STEP 4: The last step is to specify the connections between # processes, and then make and run the multiprocess app by # executing run_multiprocess() # Constants ntp_server_0 = '0.us.pool.ntp.org' ntp_server_1 = '1.us.pool.ntp.org' time_interval = 0.1 num_steps = 20 # ---------------------------------------------------------------- # MAKE PROCESS proc_0 # proc_0 has no input streams and has a single output # stream which is called 's'. # It has a single source: see source_0. # ---------------------------------------------------------------- # STEP 1: DEFINE SOURCES def source_0(out_stream): return offsets_from_ntp_server(out_stream, ntp_server_0, time_interval, num_steps) # STEP 2: DEFINE THE COMPUTATIONAL NETWORK OF AGENTS # This network is empty; it merely passes its in_stream to its # out_stream. def compute(in_streams, out_streams): map_element(func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 3: CREATE THE PROCESS # This process has a single source, no input stream, and an output # stream called 's' proc_0 = distributed_process(compute_func=compute, in_stream_names=['in'], out_stream_names=['s'], connect_sources=[('in', source_0)], name='process_1') # ---------------------------------------------------------------- # MAKE PROCESS proc_1 # proc_1 has no input streams and has a single output # stream which is called 's'. # It has a single source: see source_1. # ---------------------------------------------------------------- # STEP 1: DEFINE SOURCES def source_1(out_stream): return offsets_from_ntp_server(out_stream, ntp_server_1, time_interval, num_steps) # STEP 2: DEFINE THE COMPUTATIONAL NETWORK OF AGENTS # This network is empty; it merely passes its in_stream to its # out_stream. def compute(in_streams, out_streams): map_element(func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 3: CREATE THE PROCESS # This process has a single source, no input stream, and an output # stream called 's' proc_1 = distributed_process(compute_func=compute, in_stream_names=['in'], out_stream_names=['s'], connect_sources=[('in', source_1)], name='process_1') # ---------------------------------------------------------------- # MAKE PROCESS proc_2 # proc_2 has two input streams and no output stream. # It has no sources. # ---------------------------------------------------------------- # STEP 1: DEFINE SOURCES # This process has no sources. # STEP 2: DEFINE COMPUTE_FUNC # The composed agent consists of two component agents: # (1) blend: an agent which blends (merges) in_streams and outputs # merged_stream, and # (2) stream_to_file: a sink agent which inputs merged_stream and # prints it. def compute(in_streams, out_streams): merged_stream = Stream('merge of two ntp server offsets') blend(func=identity, in_streams=in_streams, out_stream=merged_stream) stream_to_file(in_stream=merged_stream, filename='offsets.dat') # STEP 3: CREATE THE PROCESS # This process has no sources, two input streams, and no output # streams. We call the input streams 'u' and 'v'. proc_2 = distributed_process(compute_func=compute, in_stream_names=['u', 'v'], out_stream_names=[], connect_sources=[], name='process_2') # ---------------------------------------------------------------- # FINAL STEP: RUN APPLICATION # Specify connections: A list of 4-tuples: # (process, output stream name, process, input stream name) # ---------------------------------------------------------------- vm = Multiprocess(processes=[proc_0, proc_1, proc_2], connections=[(proc_0, 's', proc_2, 'u'), (proc_1, 's', proc_2, 'v')]) vm.run()
def single_process_single_source_example_1(): """ The application in this example consists of single process. The process has a single source and no actuator. The single source generates 1, 2, 3, 4, ..... The compute function multiplies this sequence by 10 and puts the result in the file called test.dat num_steps is the number of values output by the source. For example, if num_steps is 4 and test.dat is empty before the function is called then, test.dat will contain 10, 20, 30, 40 on separate lines. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the application by calling run_multiprocess. """ # STEP 1: DEFINE SOURCES def source(out_stream): """ A simple source which outputs 1, 2, 3,... on out_stream. """ def generate_sequence(state): return state + 1, state + 1 # Return an agent which takes 10 steps, and # sleeps for 0.1 seconds between successive steps, and # puts the next element of the sequence in stream s, # and starts the sequence with value 0. The elements on # out_stream will be 1, 2, 3, ... return source_func_to_stream(func=generate_sequence, out_stream=out_stream, time_interval=0.1, num_steps=4, state=0) # STEP 2: DEFINE ACTUATORS # This example has no actuators # STEP 3: DEFINE COMPUTE_FUNC def compute_func(in_streams, out_streams): # This is a simple example of a composed agent consisting # of two component agents where the network has a single input # stream and no output stream. # The first component agent applies function f to each element # of in_stream, and puts the result in its output stream t. # The second component agent puts values in its input stream t # on a file called test.dat. # test.dat will contain 10, 20, 30, .... def f(x): return x * 10 t = Stream() map_element(func=f, in_stream=in_streams[0], out_stream=t) stream_to_file(in_stream=t, filename='test.dat') # STEP 4: CREATE PROCESSES # This process has a single input stream that we call 'in' and it # has no output streams. We connect the source to the input stream # called 'in'. proc = distributed_process(compute_func=compute_func, in_stream_names=['in'], out_stream_names=[], connect_sources=[('in', source)], connect_actuators=[], name='proc') # FINAL STEP: RUN APPLICATION # Since this application has a single process it has no # connections between processes. vm = Multiprocess(processes=[proc], connections=[]) vm.run()
def multiprocess_example_1(): """ A simple example of a multiprocess application with two processes, proc_0 and proc_1. proc_0 has a source, no input streams and a single output stream called 's'. proc_1 has no sources, a single input stream called 't', and no output streams. The connections between processes is as follows: the output stream called 's' from proc_0 is the input stream called 't' in proc_1. The source in proc_0 generates 1, 2, 3, 4,.... and the agent in proc_0 multiplies these values by 10, and so proc_0 outputs 10, 20, 30, 40, ... on its output stream. proc_1 reads the output stream of proc_0, and its agent multiplies the elements in this stream by 200 and puts the values in a file called 'result.dat' which will contain: 2000, 4000, 6000, ... The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the application by calling run_multiprocess. """ # A helper function def increment_state(state): return state + 1, state + 1 # ---------------------------------------------------------------- # DEFINE EACH OF THE PROCESSES # ---------------------------------------------------------------- # ---------------------------------------------------------------- # MAKE PROCESS proc_0 # proc_0 has no input streams and has a single output # stream which is called 't'. # It has a single source: see source_0. # ---------------------------------------------------------------- # STEP 1: DEFINE SOURCES def source_0(out_stream): return source_func_to_stream(func=increment_state, out_stream=out_stream, time_interval=0.1, num_steps=10, state=0, window_size=1, name='source') # STEP 2: DEFINE ACTUATORS # This process has no actuators # # STEP 3: DEFINE COMPUTE_FUNC # The agent for this process is a single map_element agent. # The map element agent has a single input stream: in_streams[0], # and it has a single output stream: out_streams[0]. The elements # of the output stream are 10 times the elements of the input # stream. def compute_0(in_streams, out_streams): map_element(func=lambda x: 10 * x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES # This process has no input streams and has a single output stream # which is the stream produced by the compute_0() network of # agents, and this output stream is called 's'. It has a single # source agent: source_0(). proc_0 = distributed_process(compute_func=compute_0, in_stream_names=['in'], out_stream_names=['s'], connect_sources=[('in', source_0)], name='process_0') # ---------------------------------------------------------------- # MAKE PROCESS proc_1 # proc_1 has one input stream, called 't' and has no output # streams # It has no sources. # ---------------------------------------------------------------- # STEP 1: DEFINE SOURCES # This process has no sources; so skip this step. # # STEP 2: DEFINE ACTUATORS # This process has no actuators # # STEP 3: DEFINE COMPUTE_FUNC # This network consists of a map_element agent and # a file_to_stream agent which is a type of sink agent and which # puts the elements of result_stream on a file called 'results.dat.' # result_stream is internal to the network. def compute_1(in_streams, out_streams): result_stream = Stream('result of computation') map_element(func=lambda x: 200 * x, in_stream=in_streams[0], out_stream=result_stream) stream_to_file(in_stream=result_stream, filename='result.dat') # # STEP 4: CREATE PROCESSES # This process has a single input stream, called 't', produced by # proc_1. It has no output streams. proc_1 = distributed_process(compute_func=compute_1, in_stream_names=['t'], out_stream_names=[], connect_sources=[], name='process_1') # ---------------------------------------------------------------- # FINAL STEP: RUN APPLICATION # Specify connections: A list of 4-tuples: # (process, output stream name, process, input stream name) # ---------------------------------------------------------------- vm = Multiprocess(processes=[proc_0, proc_1], connections=[(proc_0, 's', proc_1, 't')]) vm.run()
def clock_offset_estimation_single_process_multiple_sources(): """ Another test of a single process with multiple sources and no actuators. This process merges offsets received from two ntp sources and computes their average over a moving time window, and puts the result on a file, average.dat This process has two sources, each of which receives ntp offsets from ntp servers. The composed agent consists of three component agents: (1) a component agent that merges the two sources, and (2) a component agent that computes the average of the merged stream over a window, and (3) a component sink agent that puts the averaged stream in file called 'average.dat'. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final steps: After creating all processes, specify the connections between processes and then run the application by calling run_multiprocess. """ ntp_server_0 = '0.us.pool.ntp.org' ntp_server_1 = '1.us.pool.ntp.org' time_interval = 0.1 num_steps = 20 def average_of_list(a_list): if a_list: # Remove None elements from the list a_list = [i for i in a_list if i is not None] # Handle the non-empty list. if a_list: return sum(a_list) / float(len(a_list)) # Handle the empty list return 0.0 # STEP 1: DEFINE SOURCES def source_0(out_stream): return offsets_from_ntp_server(out_stream, ntp_server_0, time_interval, num_steps) def source_1(out_stream): return offsets_from_ntp_server(out_stream, ntp_server_1, time_interval, num_steps) # STEP 2: DEFINE ACTUATORS # This process has no actuators # STEP 3: DEFINE COMPUTE_FUNC # This composed agent has two input streams, one from each # source. # It has two internal streams: merged_stream and averaged_stream. # It has 3 component agents: # (1) blend: The composed agent's two input streams feed a blend # agent which outputs merged_stream. # (2) The map_window agent reads merged_stream and outputs # averaged_stream. # (3) The stream_to_file agent inputs averaged_stream. This agent # is a sink which puts the stream into the file called # 'average.dat'. The file will contain floating point numbers that # are the averages of the specified sliding winow. def compute_func(in_streams, out_streams): merged_stream = Stream('merge of two ntp server offsets') averaged_stream = Stream('sliding window average of offsets') blend(func=lambda x: x, in_streams=in_streams, out_stream=merged_stream) map_window(func=average_of_list, in_stream=merged_stream, out_stream=averaged_stream, window_size=2, step_size=1) stream_to_file(in_stream=averaged_stream, filename='average.dat') # STEP 4: CREATE PROCESSES proc = distributed_process(compute_func=compute_func, in_stream_names=['ntp_0', 'ntp_1'], out_stream_names=[], connect_sources=[('ntp_0', source_0), ('ntp_1', source_1)], connect_actuators=[], name='proc') # FINAL STEP: RUN APPLICATION vm = Multiprocess(processes=[proc], connections=[]) vm.run()
def simple_actuator_example(): """ This example has a single source which generates the sequence: 1, 2, 3, .... It has a single actuator which gets messages from a queue and prints the message. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the application by calling run_multiprocess. """ # STEP 1: DEFINE SOURCES def sequence_of_integers(current_integer, max_integer): next_integer = current_integer + 1 if next_integer > max_integer: # return next output, next state return None, next_integer else: return next_integer, next_integer def sequence_of_integers_source(out_stream): return source_func_to_stream(func=sequence_of_integers, out_stream=out_stream, num_steps=15, window_size=1, state=0, max_integer=10) # STEP 2: DEFINE ACTUATORS def print_from_queue(q): while True: v = q.get() if v is None: # exit loop return True else: print 'next value in queue is ', v # STEP 3: DEFINE COMPUTE_FUNC def f(in_streams, out_streams): def identity(v): return v map_element(func=identity, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES proc = distributed_process(compute_func=f, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', sequence_of_integers_source)], connect_actuators=[['out', print_from_queue]]) # FINAL STEP: RUN APPLICATION # Since this application has a single process it has no # connections between processes. vm = Multiprocess(processes=[proc], connections=[]) vm.run()
def single_process_multiple_sources_example_1(): """ The application in this example consists of a single process. This process has two sources: source_0 generates 1, 2, 3, 4, ... and source_1 generates random numbers. The agent zips the two streams together and writes the result to a file called output.dat. This file will have (1, r1), (2, r2), (3, r3), ... where r1, r2,.... are random numbers. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() Final step After creating all processes, specify the connections between processes and run the application by calling run_multiprocess. """ import random # STEP 1: DEFINE SOURCES def source_0(out_stream): # A simple source which outputs 1, 2, 3, 4, .... on # out_stream. def generate_sequence(state): return state + 1, state + 1 # Return a source which takes 10 steps, and # sleeps for 0.1 seconds between successive steps, and # puts the next element of the sequence in out_stream, # and starts the sequence with value 0. The elements on # out_stream will be 1, 2, 3, ... return source_func_to_stream(func=generate_sequence, out_stream=out_stream, time_interval=0.1, num_steps=10, state=0) def source_1(out_stream): # A simple source which outputs random numbers on # out_stream. # Return a source which takes 10 steps, and sleeps for 0.1 # seconds between successive steps, and puts a random number # on out_stream at each step. return source_func_to_stream(func=random.random, out_stream=out_stream, time_interval=0.1, num_steps=10) # STEP 2: DEFINE ACTUATORS # This example has no actuators # STEP 3: DEFINE COMPUTE_FUNC def compute_func(in_streams, out_streams): # This is a simple example of a composed agent consisting # of two component agents where the composed agent has two # input streams and no output stream. # The first component agent zips the two input streams and puts # the result on its output stream t which is internal to the # network. # The second component agent puts values in its input stream t # on a file called output.dat. from sink import stream_to_file # t is an internal stream of the network t = Stream() zip_stream(in_streams=in_streams, out_stream=t) stream_to_file(in_stream=t, filename='output.dat') # STEP 4: CREATE PROCESSES proc = distributed_process(compute_func=compute_func, in_stream_names=['source_0', 'source_1'], out_stream_names=[], connect_sources=[('source_0', source_0), ('source_1', source_1)], connect_actuators=[], name='multiple source test') # FINAL STEP: RUN APPLICATION # Since this application has a single process it has no # connections between processes. vm = Multiprocess(processes=[proc], connections=[]) vm.run()
def two_process_publication_example_1(): """ The application in this example consists of two VMs. It is a small extension of def single_process_publication_example_1(). The first VM has two processes, proc_0 and proc_1. The second VM has two processes, proc_2 and proc_3. THE FIRST VM PROC_0 proc_0 has a single source and no actuator. The single source generates 1, 2, 3, 4, ..... num_steps is the number of values output by the source. The compute function has a single in_stream and a single out_stream. commpute_func merely passes its in_stream to its out_stream. PROC_1 proc_1 has no sources or actuators. Its compute function has a single in_stream and a single out_stream. compute_func multiples its input elements by 10 and puts the results on out_stream. THE SECOND VM PROC_2 proc_2 has no sources or actuators. Its compute function has a single in_stream and a single out_stream. compute_func multiplies elements in in_stream by 2 and places results on out_stream. PROC_3 proc_2 has no sources or actuators. Its compute function has a single in_stream and no out_stream. compute_func prints its in_stream. The steps for creating a process are: (1) Define the sources. In this example we have two sources, source_0 and source_1 (2) Define the actuators. In this example we have no actuators. (3) Define compute_func (4) Create the process by calling distributed_process() (5) Create a VM fter creating all processes in the VM. Do this by specifying the connections between processes within the VM and by specifying the streams published by the VM and the streams subscribed to by the VM. (6) Start the VMs. (7) Join the VMs. Skip this step if a VM is persistent. """ #------------------------ #------------------------ # VM_0 #------------------------ #------------------------ #------------------------ # proc_0 in VM_0 #------------------------ # STEP 1: DEFINE SOURCES def source(out_stream): """ A simple source which outputs 1, 2, 3,... on out_stream. """ def generate_sequence(state): return state + 1, state + 1 # Return an agent which takes 10 steps, and # sleeps for 0.1 seconds between successive steps, and # puts the next element of the sequence in stream s, # and starts the sequence with value 0. The elements on # out_stream will be 1, 2, 3, ... return source_func_to_stream(func=generate_sequence, out_stream=out_stream, time_interval=0.1, num_steps=4, state=0) # STEP 2: DEFINE ACTUATORS # This example has no actuators # STEP 3: DEFINE COMPUTE_FUNC def f(in_streams, out_streams): map_element(func=lambda x: x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES # This process has a single input stream that we call 'in' and it # has no output streams. We connect the source to the input stream # called 'in'. proc_0 = distributed_process(compute_func=f, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[('in', source)], connect_actuators=[], name='proc_0') #------------------------ # proc_1 in VM_0 #------------------------ # STEP 1: DEFINE SOURCES # skip this step since proc_1 has no sources. # STEP 2: DEFINE ACTUATORS # # skip this step since proc_1 has no actuators. # STEP 3: DEFINE COMPUTE_FUNC def g(in_streams, out_streams): map_element(func=lambda x: 10 * x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES proc_1 = distributed_process(compute_func=g, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[], connect_actuators=[], name='proc_1') # STEP 5: CREATE VM vm_0 = VM(processes=[proc_0, proc_1], connections=[(proc_0, 'out', proc_1, 'in')], publishers=[(proc_1, 'out', 'sequence')]) #------------------------ #------------------------ # VM_1 #------------------------ #------------------------ #------------------------ # proc_2 in VM_1 #------------------------ # STEP 1: DEFINE SOURCES # skip this step since proc_1 has no sources. # STEP 2: DEFINE ACTUATORS # # skip this step since proc_1 has no actuators. # STEP 3: DEFINE COMPUTE_FUNC def h(in_streams, out_streams): map_element(func=lambda x: 2 * x, in_stream=in_streams[0], out_stream=out_streams[0]) # STEP 4: CREATE PROCESSES proc_2 = distributed_process(compute_func=h, in_stream_names=['in'], out_stream_names=['out'], connect_sources=[], connect_actuators=[], name='proc_2') #------------------------ # proc_3 in VM_1 #------------------------ # STEP 1: DEFINE SOURCES # skip this step since proc_1 has no sources. # STEP 2: DEFINE ACTUATORS # # skip this step since proc_1 has no actuators. # STEP 3: DEFINE COMPUTE_FUNC def pr(in_streams, out_streams): def print_element(v): print 'stream element is ', v sink_element(func=print_element, in_stream=in_streams[0]) # STEP 4: CREATE PROCESSES proc_3 = distributed_process(compute_func=pr, in_stream_names=['in'], out_stream_names=[], connect_sources=[], connect_actuators=[], name='proc_3') # STEP 5: CREATE VM vm_1 = VM(processes=[proc_2, proc_3], connections=[(proc_2, 'out', proc_3, 'in')], subscribers=[(proc_2, 'in', 'sequence')]) # STEP 6: START PROCESSES vm_0.start() vm_1.start() # STEP 7: JOIN PROCESSES vm_0.join() vm_1.join()
def detect_large_magnitude(sensor_name, filenames): # ---------------------------------------------------------------- # COMPUTE FUNCTION f # ---------------------------------------------------------------- def compute_func(in_streams, out_streams): """ Detects anomalies in streams generated by triaxial sensors. Parameters ---------- in_streams: list of Stream in_streams is a list of 3 streams indicating measurements in e, n, and z (for east, north, vertical) directions. These streams are generated by a triaxial sensor. out_streams: list of Stream out_streams has only one element, which is a Stream of int. An element of this stream is either 1.0 or 0.0. An element is 1.0 to indicate that an anomaly was detected in in_streams and is 0.0 otherwise. """ #------------------------------------------------------------------ # DECLARE INTERNAL STREAMS #------------------------------------------------------------------ # magnitudes is a stream of magnitudes of a stream of vectors # where each vector is given by its e, n, z values. magnitudes = Stream('magnitudes') anomaly_times_before_quenching = Stream('prior quench') anomaly_times_after_quenching = out_streams[0] #---------------------------------------------------- # CREATE AGENTS #---------------------------------------------------- # This agent generates streams of magnitudes of vectors # from streams of the components of the vectors. magnitude_of_vector(in_streams, out_stream=magnitudes) # This agent generates a stream of anomalies from # streams of magnitudes. simple_anomalies( in_stream=magnitudes, out_stream=anomaly_times_before_quenching, threshold=0.005) quench( in_stream=anomaly_times_before_quenching, out_stream=anomaly_times_after_quenching, QUENCH_TIME=4) # Agents that copy streams into files for later analysis. stream_to_file(anomaly_times_after_quenching, 'local_anomalies.txt') # ---------------------------------------------------------------- # DEFINE SOURCES # ---------------------------------------------------------------- def source(filename): """ This function creates a source by reading a file of floats. The source generates an element every TIME_INTERVAL seconds and stops after NUM_STEPS number of steps if NUM_STEPS is not None and outputs the entire file if NUM_STEPS is None. Parameters ---------- filename: str name of a file """ return source_float_file( filename, time_interval=0, num_steps=None).source_func directions = ['e', 'n', 'z'] proc_0 = distributed_process( compute_func=compute_func, in_stream_names=directions, out_stream_names=['out'], connect_sources=[ (directions[i], source(filenames[i])) for i in range(len(directions))] ) vm_0 = VM( processes=[proc_0], connections=[], publishers=[(proc_0, 'out', sensor_name)]) vm_0.start()