def pika_subscriber_test():
# Agent function for process named 'p0'
def g(in_streams, out_streams):
print_stream(in_streams[0], 'x')
# Source thread target for source stream named 'x'.
def source_thread_target(procs):
def callback(ch, method, properties, body):
extend_stream(procs, data=json.loads(body), stream_name='x')
pika_subscriber = PikaSubscriber(
callback, routing_key='temperature',
exchange='publications', host='localhost')
pika_subscriber.start()
# The specification
multicore_specification = [
# Streams
[('x', 'i')],
# Processes
[{'name': 'p0', 'agent': g, 'inputs':['x'], 'sources': ['x']}]]
processes, procs = get_processes_and_procs(multicore_specification)
thread_0 = threading.Thread(target=source_thread_target, args=(procs,))
procs['p0'].threads = [thread_0]
for process in processes: process.start()
for process in processes: process.join()
for process in processes: process.terminate()
def shimmer_multicore(original_sound_list, fs):
delay = int(fs / 3)
attenuation_vector = [0.6]
def echo_agent(in_streams, out_streams):
"""
Notes
-----
For this particular function, there is only
one input stream, which is the output of the pitchshifted
agent. The output stream is the generated echo.
"""
print('echo_agent')
print('type(in_streams[0]) is ', type(in_streams[0]))
print('type(out_streams[0]) is ', type(out_streams[0]))
window_dot_product(in_stream=in_streams[0],
out_stream=out_streams[0],
multiplicand_vector=attenuation_vector)
def pitch_shift(in_streams, out_streams):
"""
Notes
-----
For this particular function, there is only one
input stream, that is the output of the shimmer effect.
The output of this function is the pitchshifted version of the
heard stream.
"""
print('pitch_shift')
print('type(in_streams[0]) is ', type(in_streams[0]))
print('type(out_streams[0]) is ', type(out_streams[0]))
window_size = 2**13
h = 2**11
n = 12
factor = 2**(1.0 * n / 12.0)
f = 1.0 / factor
# Define the stretch agent
y = StreamArray('y', dtype=np.int16)
# The below is the Pitch Shift Agent
stretch_object = Stretch(in_stream=in_streams[0],
out_stream=y,
factor=factor,
window_size=window_size,
h=h)
sink_window(func=stretch_object.stretch,
in_stream=in_streams[0],
window_size=window_size + h,
step_size=int(h * f))
# Define the speedup agent.
def f(window, out_stream):
indices = np.arange(0, window_size, factor)
out_stream.extend(
np.int16(np.interp(indices, np.arange(window_size), window)) +
0.0)
sink_window(func=f,
in_stream=y,
window_size=window_size,
step_size=window_size,
out_stream=out_streams[0])
def sum_echo_input(in_streams, out_streams):
"""
Notes
-----
This function takes as input two streams: the input stream and the output of the
echo agent - i.e., the echo of the pitchshifted signal. It outputs
one stream - the heard sound.
"""
in_streams[1].extend(np.zeros(delay))
zip_map(func=sum, in_streams=in_streams, out_stream=out_streams[0])
def write_file(in_streams, out_streams):
stream_to_file(in_stream=in_streams[0], filename='output.txt')
## def get_source_from_data(proc):
## proc.copy_stream(data = original_sound_list,
## stream_name='Input Stream')
## proc.finished_source(stream_name='Input Stream')
def source_thread_target(procs):
extend_stream(procs,
data=original_sound_list,
stream_name='original_sound')
terminate_stream(procs, stream_name='original_sound')
multicore_specification = [
# Streams
[('original_sound', 'f'), ('sound_with_echo', 'f'),
('pitch_shifted_sound', 'f'), ('echo', 'f')],
# Processes
[ # Make echo from pitch_shifted_sound
{
'name': 'Echo',
'agent': echo_agent,
'inputs': ['pitch_shifted_sound'],
'outputs': ['echo']
},
# Make pitch_shifted_sound from sound_with_echo
{
'name': 'PitchShift',
'agent': pitch_shift,
'inputs': ['sound_with_echo'],
'outputs': ['pitch_shifted_sound']
},
# Make sound_with_echo from original_sound and echo.
{
'name': 'Sum_Echo_And_Input',
'agent': sum_echo_input,
'inputs': ['original_sound', 'echo'],
'outputs': ['sound_with_echo'],
'sources': ['original_sound']
},
# Write pitch_shifted_sound to file.
{
'name': 'Write',
'agent': write_file,
'inputs': ['pitch_shifted_sound']
}
]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['Sum_Echo_And_Input'].threads = [source_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print('FINISHED MULTICORE')
def twitter_analysis(consumer_key, consumer_secret, access_token,
access_token_secret):
# Agent function for process named 'TrumpProcess'
def f(in_streams, out_streams):
s = Stream('s')
t = Stream('t')
map_element(get_tweet_fields,
in_streams[0],
out_stream=s,
fields={
'-': ['text', 'created_at', 'sentiment'],
'user': [
'name', 'screen_name', 'description',
'favorites_count', 'followers_count'
]
})
filter_element(
filter_tweet,
in_stream=s,
out_stream=out_streams[0],
required_phrases=['Biden'],
avoid_phrases=['#MAGA', '@thedemocrat', 'stable genius'])
# Agent function for process named 'CovidProcess'
def g(in_streams, out_streams):
s = Stream('s')
t = Stream('t')
map_element(get_tweet_fields,
in_streams[0],
out_stream=s,
fields={
'-': ['text', 'sentiment'],
'user': [
'name', 'screen_name', 'description',
'favorites_count', 'followers_count'
]
})
filter_element(filter_tweet,
in_stream=s,
out_stream=out_streams[0],
required_phrases=['covid', 'vaccine', 'Fauci', 'Birks'],
avoid_phrases=['evil'])
# Agent function for process named 'FuseTweets'
def h(in_streams, out_streams):
s = Stream('s')
weave(in_streams, out_stream=s)
sink_element(print_tweets, s)
multicore_specification = [
# Streams
[('TrumpStream', 'x'), ('CovidStream', 'x'), ('TrumpFiltered', 'x'),
('CovidFiltered', 'x')],
# Processes
[{
'name': 'TrumpProcess',
'agent': f,
'inputs': ['TrumpStream'],
'sources': ['TrumpStream'],
'outputs': ['TrumpFiltered']
}, {
'name': 'CovidProcess',
'agent': g,
'inputs': ['CovidStream'],
'sources': ['CovidStream'],
'outputs': ['CovidFiltered']
}, {
'name': 'FuseTweets',
'agent': h,
'inputs': ['TrumpFiltered', 'CovidFiltered']
}]
]
# PROCESSES
processes, procs = get_processes_and_procs(multicore_specification)
# SOURCE THREADS
source_thread_Trump = twitter_to_stream(consumer_key,
consumer_secret,
access_token,
access_token_secret,
trackwords=['Trump'],
stream_name='TrumpStream',
procs=procs,
num_tweets=50)
source_thread_Covid = twitter_to_stream(consumer_key,
consumer_secret,
access_token,
access_token_secret,
trackwords=['Covid'],
stream_name='CovidStream',
procs=procs,
num_tweets=20)
procs['TrumpProcess'].threads = [source_thread_Trump]
procs['CovidProcess'].threads = [source_thread_Covid]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
def test_parameter_result(self):
"""
This example illustrates how you can get results from IoTPy processes when the
processes terminate. The results are stored in a buffer (a multiprocessing.Array)
which your non-IoTPy code can read. You can insert data into the IoTPy processes
continuously or before the processes are started.
In this example output_buffer[j] = 0 + 1 + 2 + ... + j
"""
print('starting test_parameter_result')
print('')
print('Output stream s and output_buffer.')
print('output_buffer is [0, 1, 3, 6, 10, .., 45]')
print('s[j] = output_buffer[j] + 100')
print('')
# The results of the parallel computation are stored in output_buffer.
output_buffer = multiprocessing.Array('i', 20)
# The results are in output_buffer[:output_buffer_ptr]
output_buffer_ptr = multiprocessing.Value('i', 0)
# In this example v represents an element of an input stream.
# sum is the sum of all the stream-element values received
# by the agent. The state of the agent is sum.
# output_buffer and output_buffer_ptr are keyword arguments.
@map_e
def ff(v, sum, output_buffer, output_buffer_ptr):
sum += v
output_buffer[output_buffer_ptr.value] = sum
output_buffer_ptr.value += 1
return sum, sum
# Agent function for process named 'p0'
def f(in_streams, out_streams, output_buffer, output_buffer_ptr):
ff(in_streams[0],
out_streams[0],
state=0,
output_buffer=output_buffer,
output_buffer_ptr=output_buffer_ptr)
# Agent function for process named 'p1'
def g(in_streams, out_streams):
s = Stream('s')
map_element(lambda v: v * 2, in_streams[0], s)
print_stream(s, 's')
# Source thread target for source stream named 'x'.
def source_thread_target(procs):
for i in range(3):
extend_stream(procs,
data=list(range(i * 2, (i + 1) * 2)),
stream_name='x')
time.sleep(0.001)
terminate_stream(procs, stream_name='x')
# Specification
multicore_specification = [
# Streams
[('x', 'i'), ('y', 'i')],
# Processes
[{
'name': 'p0',
'agent': f,
'inputs': ['x'],
'outputs': ['y'],
'keyword_args': {
'output_buffer': output_buffer,
'output_buffer_ptr': output_buffer_ptr
},
'sources': ['x']
}, {
'name': 'p1',
'agent': g,
'inputs': ['y'],
}]
]
# Execute processes (after including your own non IoTPy processes)
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['p0'].threads = [source_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
# Verify that output_buffer can be read by the parent process.
print('output_buffer is ', output_buffer[:output_buffer_ptr.value])
print('')
print(' ')
print('finished test_parameter_result')
print(' ')
def twitter_parallel(consumer_key, consumer_secret, access_token,
access_token_secret, trackwords, num_steps):
def copy_input_to_output(in_streams, out_streams):
map_element(lambda x: x, in_streams[0], out_streams[0])
def output_tweet_sentiment(in_streams, out_streams):
def get_sentiment(tweet):
return sentiment_of_text(get_text(tweet))
map_element(get_sentiment, in_streams[0], out_streams[0])
print_stream(out_streams[0], 'sentiment')
def output_followers_retweets(in_streams, out_streams):
map_element(followers_and_retweets_of_tweet, in_streams[0],
out_streams[0])
print_stream(out_streams[0], 'followers')
def zip_everything_and_file(in_streams, out_streams):
t = Stream()
zip_stream(in_streams, out_stream=t)
print_stream(t, 'zipped')
stream_to_file(in_stream=t, filename='result.dat')
multicore_specification = [
# Streams
[('source', 'x'), ('raw_tweets', 'x'), ('tweet_sentiment', 'x'),
('followers_and_retweets', 'x')],
# Processes
[{
'name': 'get_raw_tweets',
'agent': copy_input_to_output,
'inputs': ['source'],
'outputs': ['raw_tweets'],
'sources': ['source']
}, {
'name': 'get_tweet_sentiment',
'agent': output_tweet_sentiment,
'inputs': ['raw_tweets'],
'outputs': ['tweet_sentiment']
}, {
'name': 'get_followers_and_retweets',
'agent': output_followers_retweets,
'inputs': ['raw_tweets'],
'outputs': ['followers_and_retweets']
}, {
'name': 'put_results_in_file',
'agent': zip_everything_and_file,
'inputs': ['tweet_sentiment', 'followers_and_retweets']
}]
]
# PROCESSES
processes, procs = get_processes_and_procs(multicore_specification)
stream_name = 'source'
get_tweets_thread = twitter_to_stream(consumer_key, consumer_secret,
access_token, access_token_secret,
trackwords, stream_name, procs,
num_steps)
procs['get_raw_tweets'].threads = [get_tweets_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
def test_example_echo_single_core(self):
print(' ')
print('starting test_example_echo_single_core')
print(' ')
# This is the delay from when the made sound hits a
# reflecting surface.
delay = 4
# This is the attenuation of the reflected wave.
attenuation = 0.5
# The results are put in this queue. A thread reads this
# queue and feeds a speaker or headphone.
q = multiprocessing.Queue()
# Agent function for process named 'p0'
def f_echo(in_streams, out_streams, delay, attenuation, q):
echo = StreamArray('echo',
initial_value=np.array([0.0] * delay,
dtype='float'),
dtype='float')
#Note: sound_made = in_streams[0]
sound_heard = in_streams[0] + echo
map_element(lambda v: v * attenuation, sound_heard, echo)
stream_to_queue(sound_heard, q)
def source_thread_target(procs):
extend_stream(procs,
data=np.arange(10, dtype='float'),
stream_name='sound_made')
time.sleep(0.0001)
extend_stream(procs=procs,
data=np.zeros(10, dtype='float'),
stream_name='sound_made')
terminate_stream(procs, stream_name='sound_made')
# Thread that gets data from the output queue
# This thread is included in 'threads' in the specification.
# Thread target
def get_data_from_output_queue(q):
finished_getting_output = False
while not finished_getting_output:
v = q.get()
if v == '_finished': break
print('heard sound = spoken + echo: ', v)
multicore_specification = [
# Streams
[('sound_made', 'f')],
# Processes
[{
'name': 'p0',
'agent': f_echo,
'inputs': ['sound_made'],
'args': [delay, attenuation, q],
'sources': ['sound_made'],
'output_queues': [q]
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
output_thread = threading.Thread(target=get_data_from_output_queue,
args=(q, ))
procs['p0'].threads = [source_thread, output_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_example_echo_single_core')
print(' ')
def test_parameter(self, ADDEND_VALUE=500):
"""
Illustrates the use of args which is also illustrated in test_1.
This example is a small modification of test_0_0.
"""
print(' ')
print('starting test_parameter')
print(' ')
# Agent function for process named 'p0'
# ADDEND is a positional argument of f in the spec for p0.
def f(in_streams, out_streams, ADDEND):
map_element(lambda v: v + ADDEND, in_streams[0], out_streams[0])
# Agent function for process named 'p1'
def g(in_streams, out_streams):
s = Stream('s')
map_element(lambda v: v * 2, in_streams[0], s)
print_stream(s, 's')
def source_thread_target(procs):
for i in range(3):
extend_stream(procs,
data=list(range(i * 2, (i + 1) * 2)),
stream_name='x')
time.sleep(0.001)
terminate_stream(procs, stream_name='x')
multicore_specification = [
# Streams
[('x', 'i'), ('y', 'i')],
# Processes
[{
'name': 'p0',
'agent': f,
'inputs': ['x'],
'outputs': ['y'],
'args': [ADDEND_VALUE],
'sources': ['x']
}, {
'name': 'p1',
'agent': g,
'inputs': ['y']
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['p0'].threads = [source_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_parameter')
print(' ')
def test_example_output_thread_with_queue(self):
print(' ')
print('starting test_example_output_thread_with_queue')
print(' ')
q = multiprocessing.Queue()
def f(in_streams, out_streams):
map_element(lambda v: v + 100, in_streams[0], out_streams[0])
def g(in_streams, out_streams, q):
s = Stream('s')
map_element(lambda v: v * 2, in_streams[0], s)
stream_to_queue(s, q, name='copy_stream_s_to_queue_q')
def source_thread_target(procs):
for i in range(3):
extend_stream(procs,
data=list(range(i * 2, (i + 1) * 2)),
stream_name='x')
time.sleep(0.001)
terminate_stream(procs, stream_name='x')
def get_data_from_output_queue(q):
while True:
v = q.get()
if v == '_finished': break
else: print('q.get() = ', v)
multicore_specification = [
# Streams
[('x', 'i'), ('y', 'i')],
# Processes
[{
'name': 'p0',
'agent': f,
'inputs': ['x'],
'outputs': ['y'],
'sources': ['x']
}, {
'name': 'p1',
'agent': g,
'inputs': ['y'],
'args': [q],
'output_queues': [q]
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
output_thread = threading.Thread(target=get_data_from_output_queue,
args=(q, ))
procs['p0'].threads = [source_thread, output_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_example_output_thread_with_queue')
print(' ')
def test_example_echo_two_cores(self):
print(' ')
print('starting test_example_echo_two_cores')
print(' ')
# This is the delay from when the made sound hits a
# reflecting surface.
delay = 4
# This is the attenuation of the reflected wave.
attenuation = 0.5
# The results are put in this queue. A thread reads this
# queue and feeds a speaker or headphone.
q = multiprocessing.Queue()
# Agent function for process named 'p0'
# echo is a delay of zeroes followed by attenuated heard sound.
# out_streams[0], which is the same as sound_heard is
# echo + sound_made
def f_echo(in_streams, out_streams, delay):
sound_made, attenuated = in_streams
echo = Stream('echo')
echo.extend([0] * delay)
map_element(lambda v: v, attenuated, echo)
# The zip_map output is the sound heard which is
# the sound heard plus the echo.
zip_map(sum, [sound_made, echo], out_streams[0])
# Agent function for process named 'p1'
# This process puts the sound heard into the output queue
# and returns an attenuated version of the sound_heard as
# its output stream.
def g_echo(in_streams, out_streams, attenuation, q):
def gg(v):
# v is the sound heard
q.put(v)
# v*attenuation is the echo
return v * attenuation
map_element(gg, in_streams[0], out_streams[0])
def source_thread_target(procs):
data = list(range(10))
extend_stream(procs,
data=list(range(10)),
stream_name='sound_made')
time.sleep(0.0001)
extend_stream(procs, data=[0] * 10, stream_name='sound_made')
terminate_stream(procs, stream_name='sound_made')
# Thread that gets data from the output queue
# This thread is included in 'threads' in the specification.
# Thread target
def get_data_from_output_queue(q):
finished_getting_output = False
while not finished_getting_output:
v = q.get()
if v == '_finished': break
print('heard sound = spoken + echo: ', v)
multicore_specification = [
# Streams
[('sound_made', 'f'), ('attenuated', 'f'), ('sound_heard', 'f')],
# Processes
[{
'name': 'p0',
'agent': f_echo,
'inputs': ['sound_made', 'attenuated'],
'outputs': ['sound_heard'],
'keyword_args': {
'delay': delay
},
'sources': ['sound_made']
}, {
'name': 'p1',
'agent': g_echo,
'inputs': ['sound_heard'],
'outputs': ['attenuated'],
'args': [attenuation, q],
'output_queues': [q]
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
output_thread = threading.Thread(target=get_data_from_output_queue,
args=(q, ))
procs['p0'].threads = [source_thread, output_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_example_echo_two_cores')
print(' ')
def test_simple_multicore(self):
print(' ')
print('starting test_simple_multicore')
print(' ')
def f(in_streams, out_streams):
map_element(lambda v: v + 100, in_streams[0], out_streams[0])
def g(in_streams, out_streams):
s = Stream('s')
map_element(lambda v: v * 2, in_streams[0], s)
print_stream(s, 's')
def h(in_streams, out_streams):
map_element(lambda v: v * 2, in_streams[0], out_streams[0])
def r(in_streams, out_streams):
t = Stream('t')
map_element(lambda v: v * 3, in_streams[0], t)
print_stream(t, 't')
def source_thread_target(procs):
for i in range(3):
extend_stream(procs,
data=list(range(i * 2, (i + 1) * 2)),
stream_name='x')
time.sleep(0.001)
terminate_stream(procs, stream_name='x')
multicore_specification = [
# Streams
[('x', 'i'), ('y', 'i')],
# Processes
[{
'name': 'p0',
'agent': f,
'inputs': ['x'],
'outputs': ['y'],
'sources': ['x']
}, {
'name': 'p1',
'agent': g,
'inputs': ['y']
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
thread_0 = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['p1'].threads = [thread_0]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_simple_multicore')
print(' ')
def test_example_passing_data_to_multicore(self):
print(' ')
print('starting test_example_passing_data_to_multicore')
print(' ')
total = multiprocessing.Value('f')
num = multiprocessing.Value('i')
# Values computed from an earlier computation which is not shown.
# total and num are passed to the multiprocessing block.
total.value = 4.0e-13
num.value = 25
def sine(in_streams, out_streams):
map_element(np.sin,
dtype_float(in_streams[0]),
out_streams[0],
name='sine')
def cosine(in_streams, out_streams):
map_element(np.cos,
dtype_float(in_streams[0]),
out_streams[0],
name='cosine')
def tangent(in_streams, out_streams):
map_element(np.tan,
dtype_float(in_streams[0]),
out_streams[0],
name='tangent')
def coordinate(in_streams, out_streams, total, num):
x, sines, cosines, tangents = in_streams
def f(lst):
return lst[0] / lst[1]
def g(lst):
error_squared = (lst[0] - lst[1])**2
return error_squared
ratios = Stream('ratios')
errors = Stream('errors')
zip_map(f, [sines, cosines], ratios, name='sine / cosine')
zip_map(g, [ratios, tangents], errors, name='compute error')
print_stream(errors, 'error')
# Source thread target.
def source_thread_target(procs):
extend_stream(procs,
data=np.linspace(0.0, np.pi, 10),
stream_name='x')
terminate_stream(procs, stream_name='x')
multicore_specification = [
# Streams
[('x', 'f'), ('sines', 'f'), ('cosines', 'f'), ('tangents', 'f')],
# Processes
[{
'name': 'sine',
'agent': sine,
'inputs': ['x'],
'outputs': ['sines']
}, {
'name': 'cosine',
'agent': cosine,
'inputs': ['x'],
'outputs': ['cosines']
}, {
'name': 'tanget',
'agent': tangent,
'inputs': ['x'],
'outputs': ['tangents']
}, {
'name': 'coordinator',
'agent': coordinate,
'inputs': ['x', 'sines', 'cosines', 'tangents'],
'sources': ['x'],
'keyword_args': {
'total': total,
'num': num
}
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
thread_0 = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['coordinator'].threads = [thread_0]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_example_passing_data_to_multicore')
print(' ')
def test_example_merging_streams_from_multiple_processes(self):
print(' ')
print('starting test_example_merging_streams_from_multiple_processes')
print(' ')
def sine(in_streams, out_streams):
map_element(np.sin,
dtype_float(in_streams[0]),
out_streams[0],
name='sine')
def cosine(in_streams, out_streams):
map_element(np.cos,
dtype_float(in_streams[0]),
out_streams[0],
name='cosine')
def tangent(in_streams, out_streams):
map_element(np.tan,
dtype_float(in_streams[0]),
out_streams[0],
name='tangent')
def coordinate(in_streams, out_streams):
x, sines, cosines, tangents = in_streams
def f(lst):
return lst[0] / lst[1]
def g(lst):
error_squared = (lst[0] - lst[1])**2
return error_squared
ratios = Stream('ratios')
errors = Stream('errors')
zip_map(f, [sines, cosines], ratios, name='sine / cosine')
zip_map(g, [ratios, tangents], errors, name='compute error')
print_stream(errors, 'error')
# Source thread target.
def source_thread_target(procs):
extend_stream(procs,
data=np.linspace(0.0, np.pi, 10),
stream_name='x')
terminate_stream(procs, stream_name='x')
multicore_specification = [
# Streams
[('x', 'f'), ('sines', 'f'), ('cosines', 'f'), ('tangents', 'f')],
# Processes
[{
'name': 'sine',
'agent': sine,
'inputs': ['x'],
'outputs': ['sines']
}, {
'name': 'cosine',
'agent': cosine,
'inputs': ['x'],
'outputs': ['cosines']
}, {
'name': 'tanget',
'agent': tangent,
'inputs': ['x'],
'outputs': ['tangents']
}, {
'name': 'coordinator',
'agent': coordinate,
'inputs': ['x', 'sines', 'cosines', 'tangents'],
'sources': ['x']
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
thread_0 = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['coordinator'].threads = [thread_0]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_example_merging_streams_from_multiple_processes')
print(' ')
def test_multicore_with_arrays(self):
print(' ')
print('starting test_multicore_with_arrays')
print(' ')
def f_numpy(in_streams, out_streams):
map_window(np.mean,
dtype_float(in_streams[0]),
out_streams[0],
window_size=2,
step_size=2)
def g_numpy(in_streams, out_streams):
t = StreamArray('t')
map_window(max,
dtype_float(in_streams[0]),
t,
window_size=2,
step_size=2)
print_stream(t, 't')
def thread_target_numpy(procs):
for i in range(3):
extend_stream(procs,
data=list(range(i * 10, (i + 1) * 10)),
stream_name='x')
time.sleep(0.001)
terminate_stream(procs, stream_name='x')
multicore_specification = [
# Streams
[('x', 'i'), ('y', 'f')],
# Processes
[{
'name': 'p0',
'agent': f_numpy,
'inputs': ['x'],
'outputs': ['y'],
'sources': ['x']
}, {
'name': 'p1',
'agent': g_numpy,
'inputs': ['y']
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
thread_0 = threading.Thread(target=thread_target_numpy, args=(procs, ))
procs['p1'].threads = [thread_0]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print(' ')
print('finished test_multicore_with_arrays')
print(' ')
def test_example_parameters_with_queue(self,
DATA=list(range(4)),
ADDEND=10,
MULTIPLICAND=3,
EXPONENT=2):
"""
This example illustrates integrating processes running non-IoTPy
code with processes running IoTPy. The example shows how
results generated by IoTPy processes are obtained continuously
by non-IoTPy processes through queues. The example also shows
how results computed by IoTPy processes are returned to
the non-IoTPy calling process when the IoTPy processes terminate
In this simple example,
(s[j]+ADDEND)*MULTIPLICAND is the j-th value put in the queue, and
(s[j]+ADDEND)**EXPONENT is the j-th element of the buffer returned
by the multiprocess computation.
"""
print(' ')
print('starting test_example_parameters_with_queue')
print(' ')
# Values generated continuously by the IoTPy process are read by
# the calling non-IoTPy process using this queue.
q = multiprocessing.Queue()
# The results of the parallel computation are stored in buffer.
buffer = multiprocessing.Array('f', 10)
# The results are in buffer[:ptr].
# The values in buffer[ptr:] are arbitrary
ptr = multiprocessing.Value('i', 0)
# The computational function for process p0.
# Arguments are: in_streams, out_streams, and additional
# arguments. Here ADDEND is an additional argument.
def f(in_streams, out_streams, ADDEND):
map_element(lambda a: a + ADDEND, in_streams[0], out_streams[0])
# The computational function for process p1
def g(in_streams, out_streams, MULTIPLICAND, EXPONENT, q, buffer, ptr):
@sink_e
def h(v):
q.put(v * MULTIPLICAND)
buffer[ptr.value] = v**EXPONENT
ptr.value += 1
h(in_streams[0])
def source_thread_target(procs):
extend_stream(procs, data=DATA, stream_name='data')
terminate_stream(procs, stream_name='data')
multicore_specification = [
# Streams
[('data', 'f'), ('result', 'f')],
# Processes
[{
'name': 'p0',
'agent': f,
'inputs': ['data'],
'outputs': ['result'],
'args': [ADDEND],
'sources': ['data']
}, {
'name': 'p1',
'agent': g,
'inputs': ['result'],
'args': [MULTIPLICAND, EXPONENT, q, buffer, ptr],
'output_queues': [q]
}]
]
processes, procs = get_processes_and_procs(multicore_specification)
source_thread = threading.Thread(target=source_thread_target,
args=(procs, ))
procs['p0'].threads = [source_thread]
for process in processes:
process.start()
for process in processes:
process.join()
for process in processes:
process.terminate()
print('TERMINATED')
def get_data_from_output_queue(q):
queue_index = 0
while True:
v = q.get()
if v == '_finished': break
else:
print('q.get(', queue_index, ') = ', v)
queue_index += 1
get_data_from_output_queue(q)
# finished_source indicates that this source is finished. No more data will be sent on the
# stream called stream_name ('data') in the process with the specified name ('p0').
## queue_index = 0
## finished_getting_output = False
## while not finished_getting_output:
## element_from_queue = q.get()
## print ('element_from_queue is ', element_from_queue)
## print ('queue[', queue_index, '] = ', element_from_queue)
## queue_index += 1
## if element_from_queue == '_finished':
## print ('element_from_queue is finished')
## finished_getting_output = True
## break
# Get the results returned in the buffer.
print('buffer is ', buffer[:ptr.value])
print(' ')
print('finished test_example_parameters_with_queue')
print(' ')