def pika_subscriber_test(): # Agent function for process named 'p0' def g(in_streams, out_streams): print_stream(in_streams[0], 'x') # Source thread target for source stream named 'x'. def source_thread_target(procs): def callback(ch, method, properties, body): extend_stream(procs, data=json.loads(body), stream_name='x') pika_subscriber = PikaSubscriber( callback, routing_key='temperature', exchange='publications', host='localhost') pika_subscriber.start() # The specification multicore_specification = [ # Streams [('x', 'i')], # Processes [{'name': 'p0', 'agent': g, 'inputs':['x'], 'sources': ['x']}]] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=source_thread_target, args=(procs,)) procs['p0'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate()
def shimmer_multicore(original_sound_list, fs): delay = int(fs / 3) attenuation_vector = [0.6] def echo_agent(in_streams, out_streams): """ Notes ----- For this particular function, there is only one input stream, which is the output of the pitchshifted agent. The output stream is the generated echo. """ print('echo_agent') print('type(in_streams[0]) is ', type(in_streams[0])) print('type(out_streams[0]) is ', type(out_streams[0])) window_dot_product(in_stream=in_streams[0], out_stream=out_streams[0], multiplicand_vector=attenuation_vector) def pitch_shift(in_streams, out_streams): """ Notes ----- For this particular function, there is only one input stream, that is the output of the shimmer effect. The output of this function is the pitchshifted version of the heard stream. """ print('pitch_shift') print('type(in_streams[0]) is ', type(in_streams[0])) print('type(out_streams[0]) is ', type(out_streams[0])) window_size = 2**13 h = 2**11 n = 12 factor = 2**(1.0 * n / 12.0) f = 1.0 / factor # Define the stretch agent y = StreamArray('y', dtype=np.int16) # The below is the Pitch Shift Agent stretch_object = Stretch(in_stream=in_streams[0], out_stream=y, factor=factor, window_size=window_size, h=h) sink_window(func=stretch_object.stretch, in_stream=in_streams[0], window_size=window_size + h, step_size=int(h * f)) # Define the speedup agent. def f(window, out_stream): indices = np.arange(0, window_size, factor) out_stream.extend( np.int16(np.interp(indices, np.arange(window_size), window)) + 0.0) sink_window(func=f, in_stream=y, window_size=window_size, step_size=window_size, out_stream=out_streams[0]) def sum_echo_input(in_streams, out_streams): """ Notes ----- This function takes as input two streams: the input stream and the output of the echo agent - i.e., the echo of the pitchshifted signal. It outputs one stream - the heard sound. """ in_streams[1].extend(np.zeros(delay)) zip_map(func=sum, in_streams=in_streams, out_stream=out_streams[0]) def write_file(in_streams, out_streams): stream_to_file(in_stream=in_streams[0], filename='output.txt') ## def get_source_from_data(proc): ## proc.copy_stream(data = original_sound_list, ## stream_name='Input Stream') ## proc.finished_source(stream_name='Input Stream') def source_thread_target(procs): extend_stream(procs, data=original_sound_list, stream_name='original_sound') terminate_stream(procs, stream_name='original_sound') multicore_specification = [ # Streams [('original_sound', 'f'), ('sound_with_echo', 'f'), ('pitch_shifted_sound', 'f'), ('echo', 'f')], # Processes [ # Make echo from pitch_shifted_sound { 'name': 'Echo', 'agent': echo_agent, 'inputs': ['pitch_shifted_sound'], 'outputs': ['echo'] }, # Make pitch_shifted_sound from sound_with_echo { 'name': 'PitchShift', 'agent': pitch_shift, 'inputs': ['sound_with_echo'], 'outputs': ['pitch_shifted_sound'] }, # Make sound_with_echo from original_sound and echo. { 'name': 'Sum_Echo_And_Input', 'agent': sum_echo_input, 'inputs': ['original_sound', 'echo'], 'outputs': ['sound_with_echo'], 'sources': ['original_sound'] }, # Write pitch_shifted_sound to file. { 'name': 'Write', 'agent': write_file, 'inputs': ['pitch_shifted_sound'] } ] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['Sum_Echo_And_Input'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print('FINISHED MULTICORE')
def twitter_analysis(consumer_key, consumer_secret, access_token, access_token_secret): # Agent function for process named 'TrumpProcess' def f(in_streams, out_streams): s = Stream('s') t = Stream('t') map_element(get_tweet_fields, in_streams[0], out_stream=s, fields={ '-': ['text', 'created_at', 'sentiment'], 'user': [ 'name', 'screen_name', 'description', 'favorites_count', 'followers_count' ] }) filter_element( filter_tweet, in_stream=s, out_stream=out_streams[0], required_phrases=['Biden'], avoid_phrases=['#MAGA', '@thedemocrat', 'stable genius']) # Agent function for process named 'CovidProcess' def g(in_streams, out_streams): s = Stream('s') t = Stream('t') map_element(get_tweet_fields, in_streams[0], out_stream=s, fields={ '-': ['text', 'sentiment'], 'user': [ 'name', 'screen_name', 'description', 'favorites_count', 'followers_count' ] }) filter_element(filter_tweet, in_stream=s, out_stream=out_streams[0], required_phrases=['covid', 'vaccine', 'Fauci', 'Birks'], avoid_phrases=['evil']) # Agent function for process named 'FuseTweets' def h(in_streams, out_streams): s = Stream('s') weave(in_streams, out_stream=s) sink_element(print_tweets, s) multicore_specification = [ # Streams [('TrumpStream', 'x'), ('CovidStream', 'x'), ('TrumpFiltered', 'x'), ('CovidFiltered', 'x')], # Processes [{ 'name': 'TrumpProcess', 'agent': f, 'inputs': ['TrumpStream'], 'sources': ['TrumpStream'], 'outputs': ['TrumpFiltered'] }, { 'name': 'CovidProcess', 'agent': g, 'inputs': ['CovidStream'], 'sources': ['CovidStream'], 'outputs': ['CovidFiltered'] }, { 'name': 'FuseTweets', 'agent': h, 'inputs': ['TrumpFiltered', 'CovidFiltered'] }] ] # PROCESSES processes, procs = get_processes_and_procs(multicore_specification) # SOURCE THREADS source_thread_Trump = twitter_to_stream(consumer_key, consumer_secret, access_token, access_token_secret, trackwords=['Trump'], stream_name='TrumpStream', procs=procs, num_tweets=50) source_thread_Covid = twitter_to_stream(consumer_key, consumer_secret, access_token, access_token_secret, trackwords=['Covid'], stream_name='CovidStream', procs=procs, num_tweets=20) procs['TrumpProcess'].threads = [source_thread_Trump] procs['CovidProcess'].threads = [source_thread_Covid] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate()
def test_parameter_result(self): """ This example illustrates how you can get results from IoTPy processes when the processes terminate. The results are stored in a buffer (a multiprocessing.Array) which your non-IoTPy code can read. You can insert data into the IoTPy processes continuously or before the processes are started. In this example output_buffer[j] = 0 + 1 + 2 + ... + j """ print('starting test_parameter_result') print('') print('Output stream s and output_buffer.') print('output_buffer is [0, 1, 3, 6, 10, .., 45]') print('s[j] = output_buffer[j] + 100') print('') # The results of the parallel computation are stored in output_buffer. output_buffer = multiprocessing.Array('i', 20) # The results are in output_buffer[:output_buffer_ptr] output_buffer_ptr = multiprocessing.Value('i', 0) # In this example v represents an element of an input stream. # sum is the sum of all the stream-element values received # by the agent. The state of the agent is sum. # output_buffer and output_buffer_ptr are keyword arguments. @map_e def ff(v, sum, output_buffer, output_buffer_ptr): sum += v output_buffer[output_buffer_ptr.value] = sum output_buffer_ptr.value += 1 return sum, sum # Agent function for process named 'p0' def f(in_streams, out_streams, output_buffer, output_buffer_ptr): ff(in_streams[0], out_streams[0], state=0, output_buffer=output_buffer, output_buffer_ptr=output_buffer_ptr) # Agent function for process named 'p1' def g(in_streams, out_streams): s = Stream('s') map_element(lambda v: v * 2, in_streams[0], s) print_stream(s, 's') # Source thread target for source stream named 'x'. def source_thread_target(procs): for i in range(3): extend_stream(procs, data=list(range(i * 2, (i + 1) * 2)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') # Specification multicore_specification = [ # Streams [('x', 'i'), ('y', 'i')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['x'], 'outputs': ['y'], 'keyword_args': { 'output_buffer': output_buffer, 'output_buffer_ptr': output_buffer_ptr }, 'sources': ['x'] }, { 'name': 'p1', 'agent': g, 'inputs': ['y'], }] ] # Execute processes (after including your own non IoTPy processes) processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['p0'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() # Verify that output_buffer can be read by the parent process. print('output_buffer is ', output_buffer[:output_buffer_ptr.value]) print('') print(' ') print('finished test_parameter_result') print(' ')
def twitter_parallel(consumer_key, consumer_secret, access_token, access_token_secret, trackwords, num_steps): def copy_input_to_output(in_streams, out_streams): map_element(lambda x: x, in_streams[0], out_streams[0]) def output_tweet_sentiment(in_streams, out_streams): def get_sentiment(tweet): return sentiment_of_text(get_text(tweet)) map_element(get_sentiment, in_streams[0], out_streams[0]) print_stream(out_streams[0], 'sentiment') def output_followers_retweets(in_streams, out_streams): map_element(followers_and_retweets_of_tweet, in_streams[0], out_streams[0]) print_stream(out_streams[0], 'followers') def zip_everything_and_file(in_streams, out_streams): t = Stream() zip_stream(in_streams, out_stream=t) print_stream(t, 'zipped') stream_to_file(in_stream=t, filename='result.dat') multicore_specification = [ # Streams [('source', 'x'), ('raw_tweets', 'x'), ('tweet_sentiment', 'x'), ('followers_and_retweets', 'x')], # Processes [{ 'name': 'get_raw_tweets', 'agent': copy_input_to_output, 'inputs': ['source'], 'outputs': ['raw_tweets'], 'sources': ['source'] }, { 'name': 'get_tweet_sentiment', 'agent': output_tweet_sentiment, 'inputs': ['raw_tweets'], 'outputs': ['tweet_sentiment'] }, { 'name': 'get_followers_and_retweets', 'agent': output_followers_retweets, 'inputs': ['raw_tweets'], 'outputs': ['followers_and_retweets'] }, { 'name': 'put_results_in_file', 'agent': zip_everything_and_file, 'inputs': ['tweet_sentiment', 'followers_and_retweets'] }] ] # PROCESSES processes, procs = get_processes_and_procs(multicore_specification) stream_name = 'source' get_tweets_thread = twitter_to_stream(consumer_key, consumer_secret, access_token, access_token_secret, trackwords, stream_name, procs, num_steps) procs['get_raw_tweets'].threads = [get_tweets_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate()
def test_example_echo_single_core(self): print(' ') print('starting test_example_echo_single_core') print(' ') # This is the delay from when the made sound hits a # reflecting surface. delay = 4 # This is the attenuation of the reflected wave. attenuation = 0.5 # The results are put in this queue. A thread reads this # queue and feeds a speaker or headphone. q = multiprocessing.Queue() # Agent function for process named 'p0' def f_echo(in_streams, out_streams, delay, attenuation, q): echo = StreamArray('echo', initial_value=np.array([0.0] * delay, dtype='float'), dtype='float') #Note: sound_made = in_streams[0] sound_heard = in_streams[0] + echo map_element(lambda v: v * attenuation, sound_heard, echo) stream_to_queue(sound_heard, q) def source_thread_target(procs): extend_stream(procs, data=np.arange(10, dtype='float'), stream_name='sound_made') time.sleep(0.0001) extend_stream(procs=procs, data=np.zeros(10, dtype='float'), stream_name='sound_made') terminate_stream(procs, stream_name='sound_made') # Thread that gets data from the output queue # This thread is included in 'threads' in the specification. # Thread target def get_data_from_output_queue(q): finished_getting_output = False while not finished_getting_output: v = q.get() if v == '_finished': break print('heard sound = spoken + echo: ', v) multicore_specification = [ # Streams [('sound_made', 'f')], # Processes [{ 'name': 'p0', 'agent': f_echo, 'inputs': ['sound_made'], 'args': [delay, attenuation, q], 'sources': ['sound_made'], 'output_queues': [q] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) output_thread = threading.Thread(target=get_data_from_output_queue, args=(q, )) procs['p0'].threads = [source_thread, output_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_echo_single_core') print(' ')
def test_parameter(self, ADDEND_VALUE=500): """ Illustrates the use of args which is also illustrated in test_1. This example is a small modification of test_0_0. """ print(' ') print('starting test_parameter') print(' ') # Agent function for process named 'p0' # ADDEND is a positional argument of f in the spec for p0. def f(in_streams, out_streams, ADDEND): map_element(lambda v: v + ADDEND, in_streams[0], out_streams[0]) # Agent function for process named 'p1' def g(in_streams, out_streams): s = Stream('s') map_element(lambda v: v * 2, in_streams[0], s) print_stream(s, 's') def source_thread_target(procs): for i in range(3): extend_stream(procs, data=list(range(i * 2, (i + 1) * 2)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'i'), ('y', 'i')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['x'], 'outputs': ['y'], 'args': [ADDEND_VALUE], 'sources': ['x'] }, { 'name': 'p1', 'agent': g, 'inputs': ['y'] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['p0'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_parameter') print(' ')
def test_example_output_thread_with_queue(self): print(' ') print('starting test_example_output_thread_with_queue') print(' ') q = multiprocessing.Queue() def f(in_streams, out_streams): map_element(lambda v: v + 100, in_streams[0], out_streams[0]) def g(in_streams, out_streams, q): s = Stream('s') map_element(lambda v: v * 2, in_streams[0], s) stream_to_queue(s, q, name='copy_stream_s_to_queue_q') def source_thread_target(procs): for i in range(3): extend_stream(procs, data=list(range(i * 2, (i + 1) * 2)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') def get_data_from_output_queue(q): while True: v = q.get() if v == '_finished': break else: print('q.get() = ', v) multicore_specification = [ # Streams [('x', 'i'), ('y', 'i')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['x'], 'outputs': ['y'], 'sources': ['x'] }, { 'name': 'p1', 'agent': g, 'inputs': ['y'], 'args': [q], 'output_queues': [q] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) output_thread = threading.Thread(target=get_data_from_output_queue, args=(q, )) procs['p0'].threads = [source_thread, output_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_output_thread_with_queue') print(' ')
def test_example_echo_two_cores(self): print(' ') print('starting test_example_echo_two_cores') print(' ') # This is the delay from when the made sound hits a # reflecting surface. delay = 4 # This is the attenuation of the reflected wave. attenuation = 0.5 # The results are put in this queue. A thread reads this # queue and feeds a speaker or headphone. q = multiprocessing.Queue() # Agent function for process named 'p0' # echo is a delay of zeroes followed by attenuated heard sound. # out_streams[0], which is the same as sound_heard is # echo + sound_made def f_echo(in_streams, out_streams, delay): sound_made, attenuated = in_streams echo = Stream('echo') echo.extend([0] * delay) map_element(lambda v: v, attenuated, echo) # The zip_map output is the sound heard which is # the sound heard plus the echo. zip_map(sum, [sound_made, echo], out_streams[0]) # Agent function for process named 'p1' # This process puts the sound heard into the output queue # and returns an attenuated version of the sound_heard as # its output stream. def g_echo(in_streams, out_streams, attenuation, q): def gg(v): # v is the sound heard q.put(v) # v*attenuation is the echo return v * attenuation map_element(gg, in_streams[0], out_streams[0]) def source_thread_target(procs): data = list(range(10)) extend_stream(procs, data=list(range(10)), stream_name='sound_made') time.sleep(0.0001) extend_stream(procs, data=[0] * 10, stream_name='sound_made') terminate_stream(procs, stream_name='sound_made') # Thread that gets data from the output queue # This thread is included in 'threads' in the specification. # Thread target def get_data_from_output_queue(q): finished_getting_output = False while not finished_getting_output: v = q.get() if v == '_finished': break print('heard sound = spoken + echo: ', v) multicore_specification = [ # Streams [('sound_made', 'f'), ('attenuated', 'f'), ('sound_heard', 'f')], # Processes [{ 'name': 'p0', 'agent': f_echo, 'inputs': ['sound_made', 'attenuated'], 'outputs': ['sound_heard'], 'keyword_args': { 'delay': delay }, 'sources': ['sound_made'] }, { 'name': 'p1', 'agent': g_echo, 'inputs': ['sound_heard'], 'outputs': ['attenuated'], 'args': [attenuation, q], 'output_queues': [q] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) output_thread = threading.Thread(target=get_data_from_output_queue, args=(q, )) procs['p0'].threads = [source_thread, output_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_echo_two_cores') print(' ')
def test_simple_multicore(self): print(' ') print('starting test_simple_multicore') print(' ') def f(in_streams, out_streams): map_element(lambda v: v + 100, in_streams[0], out_streams[0]) def g(in_streams, out_streams): s = Stream('s') map_element(lambda v: v * 2, in_streams[0], s) print_stream(s, 's') def h(in_streams, out_streams): map_element(lambda v: v * 2, in_streams[0], out_streams[0]) def r(in_streams, out_streams): t = Stream('t') map_element(lambda v: v * 3, in_streams[0], t) print_stream(t, 't') def source_thread_target(procs): for i in range(3): extend_stream(procs, data=list(range(i * 2, (i + 1) * 2)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'i'), ('y', 'i')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['x'], 'outputs': ['y'], 'sources': ['x'] }, { 'name': 'p1', 'agent': g, 'inputs': ['y'] }] ] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=source_thread_target, args=(procs, )) procs['p1'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_simple_multicore') print(' ')
def test_example_passing_data_to_multicore(self): print(' ') print('starting test_example_passing_data_to_multicore') print(' ') total = multiprocessing.Value('f') num = multiprocessing.Value('i') # Values computed from an earlier computation which is not shown. # total and num are passed to the multiprocessing block. total.value = 4.0e-13 num.value = 25 def sine(in_streams, out_streams): map_element(np.sin, dtype_float(in_streams[0]), out_streams[0], name='sine') def cosine(in_streams, out_streams): map_element(np.cos, dtype_float(in_streams[0]), out_streams[0], name='cosine') def tangent(in_streams, out_streams): map_element(np.tan, dtype_float(in_streams[0]), out_streams[0], name='tangent') def coordinate(in_streams, out_streams, total, num): x, sines, cosines, tangents = in_streams def f(lst): return lst[0] / lst[1] def g(lst): error_squared = (lst[0] - lst[1])**2 return error_squared ratios = Stream('ratios') errors = Stream('errors') zip_map(f, [sines, cosines], ratios, name='sine / cosine') zip_map(g, [ratios, tangents], errors, name='compute error') print_stream(errors, 'error') # Source thread target. def source_thread_target(procs): extend_stream(procs, data=np.linspace(0.0, np.pi, 10), stream_name='x') terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'f'), ('sines', 'f'), ('cosines', 'f'), ('tangents', 'f')], # Processes [{ 'name': 'sine', 'agent': sine, 'inputs': ['x'], 'outputs': ['sines'] }, { 'name': 'cosine', 'agent': cosine, 'inputs': ['x'], 'outputs': ['cosines'] }, { 'name': 'tanget', 'agent': tangent, 'inputs': ['x'], 'outputs': ['tangents'] }, { 'name': 'coordinator', 'agent': coordinate, 'inputs': ['x', 'sines', 'cosines', 'tangents'], 'sources': ['x'], 'keyword_args': { 'total': total, 'num': num } }] ] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=source_thread_target, args=(procs, )) procs['coordinator'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_passing_data_to_multicore') print(' ')
def test_example_merging_streams_from_multiple_processes(self): print(' ') print('starting test_example_merging_streams_from_multiple_processes') print(' ') def sine(in_streams, out_streams): map_element(np.sin, dtype_float(in_streams[0]), out_streams[0], name='sine') def cosine(in_streams, out_streams): map_element(np.cos, dtype_float(in_streams[0]), out_streams[0], name='cosine') def tangent(in_streams, out_streams): map_element(np.tan, dtype_float(in_streams[0]), out_streams[0], name='tangent') def coordinate(in_streams, out_streams): x, sines, cosines, tangents = in_streams def f(lst): return lst[0] / lst[1] def g(lst): error_squared = (lst[0] - lst[1])**2 return error_squared ratios = Stream('ratios') errors = Stream('errors') zip_map(f, [sines, cosines], ratios, name='sine / cosine') zip_map(g, [ratios, tangents], errors, name='compute error') print_stream(errors, 'error') # Source thread target. def source_thread_target(procs): extend_stream(procs, data=np.linspace(0.0, np.pi, 10), stream_name='x') terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'f'), ('sines', 'f'), ('cosines', 'f'), ('tangents', 'f')], # Processes [{ 'name': 'sine', 'agent': sine, 'inputs': ['x'], 'outputs': ['sines'] }, { 'name': 'cosine', 'agent': cosine, 'inputs': ['x'], 'outputs': ['cosines'] }, { 'name': 'tanget', 'agent': tangent, 'inputs': ['x'], 'outputs': ['tangents'] }, { 'name': 'coordinator', 'agent': coordinate, 'inputs': ['x', 'sines', 'cosines', 'tangents'], 'sources': ['x'] }] ] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=source_thread_target, args=(procs, )) procs['coordinator'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_example_merging_streams_from_multiple_processes') print(' ')
def test_multicore_with_arrays(self): print(' ') print('starting test_multicore_with_arrays') print(' ') def f_numpy(in_streams, out_streams): map_window(np.mean, dtype_float(in_streams[0]), out_streams[0], window_size=2, step_size=2) def g_numpy(in_streams, out_streams): t = StreamArray('t') map_window(max, dtype_float(in_streams[0]), t, window_size=2, step_size=2) print_stream(t, 't') def thread_target_numpy(procs): for i in range(3): extend_stream(procs, data=list(range(i * 10, (i + 1) * 10)), stream_name='x') time.sleep(0.001) terminate_stream(procs, stream_name='x') multicore_specification = [ # Streams [('x', 'i'), ('y', 'f')], # Processes [{ 'name': 'p0', 'agent': f_numpy, 'inputs': ['x'], 'outputs': ['y'], 'sources': ['x'] }, { 'name': 'p1', 'agent': g_numpy, 'inputs': ['y'] }] ] processes, procs = get_processes_and_procs(multicore_specification) thread_0 = threading.Thread(target=thread_target_numpy, args=(procs, )) procs['p1'].threads = [thread_0] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print(' ') print('finished test_multicore_with_arrays') print(' ')
def test_example_parameters_with_queue(self, DATA=list(range(4)), ADDEND=10, MULTIPLICAND=3, EXPONENT=2): """ This example illustrates integrating processes running non-IoTPy code with processes running IoTPy. The example shows how results generated by IoTPy processes are obtained continuously by non-IoTPy processes through queues. The example also shows how results computed by IoTPy processes are returned to the non-IoTPy calling process when the IoTPy processes terminate In this simple example, (s[j]+ADDEND)*MULTIPLICAND is the j-th value put in the queue, and (s[j]+ADDEND)**EXPONENT is the j-th element of the buffer returned by the multiprocess computation. """ print(' ') print('starting test_example_parameters_with_queue') print(' ') # Values generated continuously by the IoTPy process are read by # the calling non-IoTPy process using this queue. q = multiprocessing.Queue() # The results of the parallel computation are stored in buffer. buffer = multiprocessing.Array('f', 10) # The results are in buffer[:ptr]. # The values in buffer[ptr:] are arbitrary ptr = multiprocessing.Value('i', 0) # The computational function for process p0. # Arguments are: in_streams, out_streams, and additional # arguments. Here ADDEND is an additional argument. def f(in_streams, out_streams, ADDEND): map_element(lambda a: a + ADDEND, in_streams[0], out_streams[0]) # The computational function for process p1 def g(in_streams, out_streams, MULTIPLICAND, EXPONENT, q, buffer, ptr): @sink_e def h(v): q.put(v * MULTIPLICAND) buffer[ptr.value] = v**EXPONENT ptr.value += 1 h(in_streams[0]) def source_thread_target(procs): extend_stream(procs, data=DATA, stream_name='data') terminate_stream(procs, stream_name='data') multicore_specification = [ # Streams [('data', 'f'), ('result', 'f')], # Processes [{ 'name': 'p0', 'agent': f, 'inputs': ['data'], 'outputs': ['result'], 'args': [ADDEND], 'sources': ['data'] }, { 'name': 'p1', 'agent': g, 'inputs': ['result'], 'args': [MULTIPLICAND, EXPONENT, q, buffer, ptr], 'output_queues': [q] }] ] processes, procs = get_processes_and_procs(multicore_specification) source_thread = threading.Thread(target=source_thread_target, args=(procs, )) procs['p0'].threads = [source_thread] for process in processes: process.start() for process in processes: process.join() for process in processes: process.terminate() print('TERMINATED') def get_data_from_output_queue(q): queue_index = 0 while True: v = q.get() if v == '_finished': break else: print('q.get(', queue_index, ') = ', v) queue_index += 1 get_data_from_output_queue(q) # finished_source indicates that this source is finished. No more data will be sent on the # stream called stream_name ('data') in the process with the specified name ('p0'). ## queue_index = 0 ## finished_getting_output = False ## while not finished_getting_output: ## element_from_queue = q.get() ## print ('element_from_queue is ', element_from_queue) ## print ('queue[', queue_index, '] = ', element_from_queue) ## queue_index += 1 ## if element_from_queue == '_finished': ## print ('element_from_queue is finished') ## finished_getting_output = True ## break # Get the results returned in the buffer. print('buffer is ', buffer[:ptr.value]) print(' ') print('finished test_example_parameters_with_queue') print(' ')