def __init__(self,
computation,
host='',
port=8181,
poll_sec=10,
DocumentRoot=None,
keyfile=None,
certfile=None,
show_coro_args=True):
self._lock = threading.Lock()
if not DocumentRoot:
DocumentRoot = os.path.join(os.path.dirname(__file__), 'data')
self._nodes = {}
self._updates = {}
if poll_sec < 1:
asyncoro.logger.warning(
'invalid poll_sec value %s; it must be at least 1', poll_sec)
poll_sec = 1
self._poll_sec = poll_sec
self._show_args = bool(show_coro_args)
self._server = BaseHTTPServer.HTTPServer(
(host, port), lambda *args: HTTPServer._HTTPRequestHandler(
self, DocumentRoot, *args))
if certfile:
self._server.socket = ssl.wrap_socket(self._server.socket,
keyfile=keyfile,
certfile=certfile,
server_side=True)
self._httpd_thread = threading.Thread(
target=self._server.serve_forever)
self._httpd_thread.daemon = True
self._httpd_thread.start()
self.computation = computation
self.status_coro = asyncoro.Coro(self.status_proc)
if computation.status_coro:
client_coro = computation.status_coro
def chain_msgs(coro=None):
coro.set_daemon()
while 1:
msg = yield coro.receive()
self.status_coro.send(msg)
client_coro.send(msg)
computation.status_coro = asyncoro.Coro(chain_msgs)
else:
computation.status_coro = self.status_coro
asyncoro.logger.info('Started HTTP%s server at %s',
's' if certfile else '',
str(self._server.socket.getsockname()))
def create_job(i, coro=None):
# create reader and send to rcoro so it can send messages to reader
client_reader = asyncoro.Coro(get_output, i)
# schedule rcoro on (available) remote server
rcoro = yield rcoro_scheduler.schedule(rcoro_proc, client_reader,
program_path)
if isinstance(rcoro, asyncoro.Coro):
print(' job %s processed by %s' % (i, rcoro.location))
# sender sends input data to rcoro
asyncoro.Coro(send_input, rcoro)
# wait for all data to be received
yield client_reader.finish()
print(' job %s done' % i)
else: # failed to schedule
client_reader.terminate()
def client_proc(computation, njobs, coro=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
# send 5 requests to remote process (compute_coro)
def send_requests(rcoro, coro=None):
# first send this local coroutine (to whom rcoro sends result)
rcoro.send(coro)
for i in range(5):
# even if recipient doesn't use "yield" (such as executing long-run
# computation, or thread-blocking function such as 'time.sleep' as
# in this case), the message is accepted by another scheduler
# (_ReactAsynCoro_) at the receiver and put in recipient's message
# queue
rcoro.send(random.uniform(10, 20))
# assume delay in input availability
yield coro.sleep(random.uniform(2, 5))
# end of input is indicated with None
rcoro.send(None)
result = yield coro.receive() # get result
print(' %s computed result: %.4f' % (rcoro.location, result))
for i in range(njobs):
rcoro = yield computation.run(compute_coro)
if isinstance(rcoro, asyncoro.Coro):
print(' job %d processed by %s' % (i, rcoro.location))
asyncoro.Coro(send_requests, rcoro)
yield computation.close()
def client_proc(computation, njobs, coro=None):
# use RemoteCoroScheduler to schedule/submit coroutines; scheduler must be
# created before computation is scheduled (next step below)
rcoro_scheduler = RemoteCoroScheduler(computation)
# create a separate coroutine to receive results, so they can be processed
# as soon as received
def recv_results(coro=None):
for i in range(njobs):
msg = yield coro.receive()
print(' result for job %d: %s' % (i, msg))
# remote coroutines send replies as messages to this coro
results_coro = asyncoro.Coro(recv_results)
# submit njobs; each job will be executed by one discoro server
for i in range(njobs):
cobj = C(i)
cobj.n = random.uniform(5, 10)
# as noted in 'discoro_client1.py', 'schedule' method is used to run
# jobs sequentially; use 'submit' to run multiple jobs on one server
# concurrently
print(' request %d: %s' % (i, cobj.n))
rcoro = yield rcoro_scheduler.schedule(compute, cobj, results_coro)
if not isinstance(rcoro, asyncoro.Coro):
print('failed to create rcoro %s: %s' % (i, rcoro))
# wait for all results and close computation
yield rcoro_scheduler.finish(close=True)
def shutdown(self, wait=True):
"""This method should be called by user program to close the
http server. If 'wait' is True the server waits for poll_sec
so the http client gets all the updates before server is
closed.
"""
if wait:
asyncoro.logger.info(
'HTTP server waiting for %s seconds for client updates '
'before quitting', self._poll_sec)
if asyncoro.AsynCoro().cur_coro():
def _shutdown(coro=None):
yield coro.sleep(self._poll_sec + 0.5)
self._server.shutdown()
self._server.server_close()
asyncoro.Coro(_shutdown)
else:
time.sleep(self._poll_sec + 0.5)
self._server.shutdown()
self._server.server_close()
else:
self._server.shutdown()
self._server.server_close()
def rci_test(coro=None):
# if server is on remote network, automatic discovery won't work,
# so add it explicitly
# yield scheduler.peer('192.168.21.5')
# find where 'rci_1' is registered (with given name in any
# known peer)
rci1 = yield asyncoro.RCI.locate('rci_1')
print('RCI is at %s' % rci1.location)
# alternately, location can be explicitly created with
# asyncoro.Location or obtained with 'locate' of asyncoro etc.
loc = yield scheduler.locate('server')
rci1 = yield asyncoro.RCI.locate('rci_1', loc)
print('RCI is at %s' % rci1.location)
n = 5
monitor = asyncoro.Coro(monitor_proc, n)
for x in range(n):
rcoro = yield rci1('test%s' % x, b=x)
asyncoro.logger.debug('RCI %s created' % rcoro)
# set 'monitor' as monitor for this coroutine
yield monitor.monitor(rcoro)
# send a message
rcoro.send('msg:%s' % x)
yield coro.sleep(random.uniform(0, 1))
def send_proc(coro=None):
# if server is in a remote network, use 'peer' as (optionally
# enabling streaming for efficiency):
# yield asyncoro.AsynCoro.instance().peer('server node/ip')
server = yield asyncoro.Coro.locate('chat_server')
server.send(('join', coro))
client_id = yield coro.receive()
# channel is at same location as server coroutine
channel = yield asyncoro.Channel.locate('chat_channel', server.location)
recv_coro = asyncoro.Coro(recv_proc, client_id)
yield channel.subscribe(recv_coro)
# since readline is synchronous (blocking) call, use async thread
async_threads = asyncoro.AsyncThreadPool(1)
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
line = yield async_threads.async_task(read_input)
line = line.strip()
if line.lower() in ('quit', 'exit'):
break
except:
break
channel.send((line, client_id))
server.send(('quit', client_id))
yield channel.unsubscribe(recv_coro)
def client_proc(computation, njobs, coro=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
# create a separate coroutine to receive results, so they can be processed
# as soon as received
def recv_results(coro=None):
for i in range(njobs):
msg = yield coro.receive()
print(' result for job %d: %s' % (i, msg))
# remote coroutines send replies as messages to this coro
results_coro = asyncoro.Coro(recv_results)
# run njobs; each job will be executed by one discoro server
for i in range(njobs):
cobj = C(i)
cobj.n = random.uniform(5, 10)
# as noted in 'discoro_client2.py', 'run' method is used to run jobs
# sequentially; use 'run_async' to run multiple jobs on one server
# concurrently
print(' request %d: %s' % (i, cobj.n))
rcoro = yield computation.run(compute, cobj, results_coro)
if not isinstance(rcoro, asyncoro.Coro):
print('failed to create rcoro %s: %s' % (i, rcoro))
# wait for all results and close computation
yield computation.close()
def client_proc(computation, n, coro=None):
# pair EC2 node with this client with:
yield asyncoro.AsynCoro().peer(asyncoro.Location('54.204.242.185', 51347))
# if multiple nodes are used, 'broadcast' option can be used to pair with
# all nodes with just one statement as:
# yield asyncoro.AsynCoro().peer(asyncoro.Location('54.204.242.185', 51347), broadcast=True)
# coroutine to call RemoteCoroScheduler's "execute" method
def exec_proc(gen, *args, **kwargs):
# execute computation; result of computation is result of
# 'yield' which is also result of this coroutine (obtained
# with 'finish' method below)
yield job_scheduler.execute(gen, *args, **kwargs)
# results can be processed here (as they become available), or
# await in sequence as done below
# Use RemoteCoroScheduler to run at most one coroutine at a server process
# This should be created before scheduling computation
job_scheduler = RemoteCoroScheduler(computation)
# execute n jobs (coroutines) and get their results. Note that
# number of jobs created can be more than number of server
# processes available; the scheduler will use as many processes as
# necessary/available, running one job at a server process
jobs = [
asyncoro.Coro(exec_proc, compute, random.uniform(3, 10))
for _ in range(n)
]
for job in jobs:
print('result: %s' % (yield job.finish()))
yield job_scheduler.finish(close=True)
def client_proc(computation, njobs, coro=None):
# use RemoteCoroScheduler to start coroutines at servers (should be done
# before scheduling computation)
rcoro_scheduler = RemoteCoroScheduler(computation)
# send 5 requests to remote process (compute_coro)
def send_requests(rcoro, coro=None):
# first send this local coroutine (to whom rcoro sends result)
rcoro.send(coro)
for i in range(5):
# even if recipient doesn't use "yield" (such as executing long-run
# computation, or thread-blocking function such as 'time.sleep' as
# in this case), the message is accepted by another scheduler
# (_ReactAsynCoro_) at the receiver and put in recipient's message
# queue
rcoro.send(random.uniform(10, 20))
# assume delay in input availability
yield coro.sleep(random.uniform(2, 5))
# end of input is indicated with None
rcoro.send(None)
result = yield coro.receive() # get result
print(' %s computed result: %.4f' % (rcoro.location, result))
for i in range(njobs):
rcoro = yield rcoro_scheduler.schedule(compute_coro)
if isinstance(rcoro, asyncoro.Coro):
print(' job %d processed by %s' % (i, rcoro.location))
asyncoro.Coro(send_requests, rcoro)
yield rcoro_scheduler.finish(close=True)
def client_proc(computation, njobs, coro=None):
computation.status_coro = asyncoro.Coro(status_proc, computation, njobs)
# since RemoteCoroScheduler is not used, schedule computation
if (yield computation.schedule()):
raise Exception('Failed to schedule computation')
# wait for jobs to be created and finished
yield computation.status_coro.finish()
yield computation.close()
def submit_jobs_proc(computation, njobs, coro=None):
for i in range(njobs):
# create remote coroutine
rcoro = yield rcoro_scheduler.schedule(rcoro_proc)
if isinstance(rcoro, asyncoro.Coro):
# create local coroutine to send input file and data to rcoro
asyncoro.Coro(client_proc, i, rcoro)
yield rcoro_scheduler.finish(close=True)
def status_proc(coro=None):
coro.set_daemon()
i = 0
while 1:
msg = yield coro.receive()
if not isinstance(msg, DiscoroStatus):
continue
if msg.status == Scheduler.NodeDiscovered:
asyncoro.Coro(node_available, msg.info.avail_info, data_files[i])
i += 1
def submit_jobs_proc(computation, njobs, coro=None):
# use RemoteCoroScheduler to schedule/submit coroutines; scheduler must be
# created before computation is scheduled (next step below)
rcoro_scheduler = RemoteCoroScheduler(computation)
for i in range(njobs):
# create remote coroutine
rcoro = yield rcoro_scheduler.schedule(rcoro_proc)
if isinstance(rcoro, asyncoro.Coro):
# create local coroutine to send input file and data to rcoro
asyncoro.Coro(client_proc, i, rcoro)
yield rcoro_scheduler.finish(close=True)
def client_proc(computation, program_path, n, coro=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
# send 10 random numbers to remote process (rcoro_proc)
def send_input(rcoro, coro=None):
for i in range(10):
# encode strings so works with both Python 2.7 and 3
rcoro.send(('%.2f' % random.uniform(0, 5)).encode())
# assume delay in input availability
yield coro.sleep(random.uniform(0, 2))
# end of input is indicated with None
rcoro.send(None)
# read output (messages sent by 'reader_proc' on remote process)
def get_output(i, coro=None):
while True:
line = yield coro.receive()
if not line: # end of output
break
print(' job %s output: %s' % (i, line.strip().decode()))
def create_job(i, coro=None):
# create reader and send to rcoro so it can send messages to reader
client_reader = asyncoro.Coro(get_output, i)
# schedule rcoro on (available) remote server
rcoro = yield computation.run(rcoro_proc, client_reader, program_path)
if isinstance(rcoro, asyncoro.Coro):
print(' job %s processed by %s' % (i, rcoro.location))
# sender sends input data to rcoro
asyncoro.Coro(send_input, rcoro)
# wait for all data to be received
yield client_reader.finish()
print(' job %s done' % i)
else: # failed to schedule
print(' job %s failed: %s' % (i, rcoro))
client_reader.terminate()
# create n jobs (that run concurrently)
job_coros = []
for i in range(1, n + 1):
job_coros.append(asyncoro.Coro(create_job, i))
# wait for jobs to finish
for job_coro in job_coros:
yield job_coro.finish()
yield computation.close()
def client_proc(computation, coro=None):
# use RemoteCoroScheduler to schedule/submit coroutines; scheduler must be
# created before computation is scheduled (next step below)
rcoro_scheduler = RemoteCoroScheduler(computation)
# in discoro_client6.py, data is sent to each remote coroutine; here, data
# is broadcast over channel and remote coroutines subscribe to it
data_channel = asyncoro.Channel('data_channel')
# not necessary to register channel in this case, as it is sent to remote
# coroutines; if they were to 'locate' it, it should be registered
# data_channel.register()
trend_coro = asyncoro.Coro(trend_proc)
rcoro_avg = yield rcoro_scheduler.schedule(rcoro_avg_proc, data_channel,
0.4, trend_coro, 10)
assert isinstance(rcoro_avg, asyncoro.Coro)
rcoro_save = yield rcoro_scheduler.schedule(rcoro_save_proc, data_channel)
assert isinstance(rcoro_save, asyncoro.Coro)
# make sure both remote coroutines have subscribed to channel ('deliver'
# should return 2 if they both are)
assert (yield data_channel.deliver('start', n=2)) == 2
# if data is sent frequently (say, many times a second), enable
# streaming data to remote peer; this is more efficient as
# connections are kept open (so the cost of opening and closing
# connections is avoided), but keeping too many connections open
# consumes system resources
yield asyncoro.AsynCoro.instance().peer(rcoro_avg.location,
stream_send=True)
yield asyncoro.AsynCoro.instance().peer(rcoro_save.location,
stream_send=True)
# send 1000 items of random data to remote coroutines
for i in range(1000):
n = random.uniform(-1, 1)
item = (i, n)
# data can be sent to remote coroutines either with 'send' or
# 'deliver'; 'send' is more efficient but no guarantee data
# has been sent successfully whereas 'deliver' indicates
# errors right away
data_channel.send(item)
yield coro.sleep(0.02)
item = (i, None)
data_channel.send(item)
yield rcoro_scheduler.finish(close=True)
data_channel.close()
def rcoro_proc(client, program, coro=None):
import sys
import os
import subprocess
import asyncoro.asyncfile
if program.endswith('.py'):
# Computation dependencies are saved in parent directory
program = [sys.executable, os.path.join('..', program)]
# start program as a subprocess (to read from and write to pipe)
if os.name == 'nt': # create pipe with asyncfile under Windows
pipe = asyncoro.asyncfile.Popen(program,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
else:
pipe = subprocess.Popen(program,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
# convert to asynchronous pipe; see 'pipe_csum.py' and 'pipe_grep.py' for
# chaining pipes
pipe = asyncoro.asyncfile.AsyncPipe(pipe)
# reader reads (output) from pipe and sends to client as messages
def reader_proc(coro=None):
while True:
line = yield pipe.readline()
if not line:
break
# send output to client
client.send(line)
pipe.stdout.close()
if os.name == 'nt':
pipe.close()
client.send(None)
reader = asyncoro.Coro(reader_proc)
# writer gets messages from client and writes them (input) to pipe
while True:
data = yield coro.receive()
if not data:
break
# write data as lines to program
yield pipe.write(data + '\n'.encode(), full=True)
pipe.stdin.close()
# wait for all data to be read (subprocess to end)
yield reader.finish()
raise StopIteration(pipe.poll())
def run_jobs_proc(computation, data_files, coro=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
for i in range(len(data_files)):
data_file = data_files[i]
# create remote coroutine
rcoro = yield computation.run(rcoro_proc)
if isinstance(rcoro, asyncoro.Coro):
# create local coroutine to send input file and data to rcoro
asyncoro.Coro(client_proc, i, data_file, rcoro)
else:
print(' job %s failed: %s' % (i, rcoro))
yield computation.close()
def receiver_proc2(coro=None):
# if server is in remote network, add it explicitly
# yield scheduler.peer('remote.peer.ip', stream_send=True)
# get reference to remote channel in server
rchannel = yield asyncoro.Channel.locate('2clients')
# this coro subscribes to the channel, so gets messages to server
# channel
print('server is at %s' % rchannel.location)
if (yield rchannel.subscribe(coro)) != 0:
raise Exception('subscription failed')
asyncoro.Coro(sender_proc, rchannel)
while True:
msg = yield coro.receive()
if msg is None:
break
print('Received "%s" from %s at %s' % \
(msg['msg'], msg['sender'].name, msg['sender'].location))
yield rchannel.unsubscribe(coro)
def client_proc(computation, program_path, n, coro=None):
# use RemoteCoroScheduler to run jobs
rcoro_scheduler = asyncoro.discoro_schedulers.RemoteCoroScheduler(
computation)
# send 10 random numbers to remote process (rcoro_proc)
def send_input(rcoro, coro=None):
for i in range(10):
# encode strings so works with both Python 2.7 and 3
rcoro.send(('%.2f' % random.uniform(0, 5)).encode())
# assume delay in input availability
yield coro.sleep(random.uniform(0, 2))
# end of input is indicated with None
rcoro.send(None)
# read output (messages sent by 'reader_proc' on remote process)
def get_output(i, coro=None):
while True:
line = yield coro.receive()
if not line: # end of output
break
print(' job %s output: %s' % (i, line.strip().decode()))
def create_job(i, coro=None):
# create reader and send to rcoro so it can send messages to reader
client_reader = asyncoro.Coro(get_output, i)
# schedule rcoro on (available) remote server
rcoro = yield rcoro_scheduler.schedule(rcoro_proc, client_reader,
program_path)
if isinstance(rcoro, asyncoro.Coro):
print(' job %s processed by %s' % (i, rcoro.location))
# sender sends input data to rcoro
asyncoro.Coro(send_input, rcoro)
# wait for all data to be received
yield client_reader.finish()
print(' job %s done' % i)
else: # failed to schedule
client_reader.terminate()
# create n jobs
for i in range(1, n + 1):
asyncoro.Coro(create_job, i)
yield rcoro_scheduler.finish(close=True)
def client_proc(computation, coro=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
trend_coro = asyncoro.Coro(trend_proc)
# run average and save coroutines at two different servers
rcoro_avg = yield computation.run(rcoro_avg_proc, 0.4, trend_coro, 10)
assert isinstance(rcoro_avg, asyncoro.Coro)
rcoro_save = yield computation.run(rcoro_save_proc)
assert isinstance(rcoro_save, asyncoro.Coro)
# if data is sent frequently (say, many times a second), enable
# streaming data to remote peer; this is more efficient as
# connections are kept open (so the cost of opening and closing
# connections is avoided), but keeping too many connections open
# consumes system resources
yield asyncoro.AsynCoro.instance().peer(rcoro_avg.location,
stream_send=True)
yield asyncoro.AsynCoro.instance().peer(rcoro_save.location,
stream_send=True)
# send 1000 items of random data to remote coroutines
for i in range(1000):
n = random.uniform(-1, 1)
item = (i, n)
# data can be sent to remote coroutines either with 'send' or
# 'deliver'; 'send' is more efficient but no guarantee data
# has been sent successfully whereas 'deliver' indicates
# errors right away; alternately, messages can be sent with a
# channel, which is more convenient if there are multiple
# (unknown) recipients
rcoro_avg.send(item)
rcoro_save.send(item)
yield coro.sleep(0.01)
item = (i, None)
rcoro_avg.send(item)
rcoro_save.send(item)
yield computation.close()
def client_proc(computation, njobs, coro=None):
# 'status_proc' receives status messages from discoro scheduler and sends
# them to both RemoteCoroScheduler and httpd
def status_proc(coro=None):
coro.set_daemon()
while True:
msg = yield coro.receive()
# send message to RemoteCoroScheduler's status_proc:
rcoro_scheduler.status_coro.send(msg)
# and to httpd's status_coro:
httpd.status_coro.send(msg)
if isinstance(msg, asyncoro.MonitorException): # a job finished
rcoro = msg.args[0]
result = msg.args[1][1]
if msg.args[1][0] == StopIteration:
print(' result for job %s from %s: %s' %
(result[0], rcoro.location, result[1]))
else:
print(' %s failed: %s' % (rcoro.location, str(result)))
# to illustrate relaying of status messages to multiple coroutines, httpd is
# also used in this example:
httpd = asyncoro.httpd.HTTPServer(computation)
# replace computation's status_coro (from rcoro_scheduler's status_coro) to
# 'status_proc' above
computation.status_coro = asyncoro.Coro(status_proc)
# submit jobs
for i in range(njobs):
rcoro = yield rcoro_scheduler.schedule(compute, i,
random.uniform(5, 10))
if isinstance(rcoro, asyncoro.Coro):
print(' job %s processed by %s' % (i, rcoro.location))
else:
print('rcoro %s failed: %s' % (i, rcoro))
# wait for all jobs to be done and close computation
yield rcoro_scheduler.finish(close=True)
httpd.shutdown()
def client_proc(computation, coro=None):
# use RemoteCoroScheduler to schedule/submit coroutines; scheduler must be
# created before computation is scheduled (next step below)
rcoro_scheduler = RemoteCoroScheduler(computation)
trend_coro = asyncoro.Coro(trend_proc)
rcoro_avg = yield rcoro_scheduler.schedule(rcoro_avg_proc, 0.4, trend_coro,
10)
assert isinstance(rcoro_avg, asyncoro.Coro)
rcoro_save = yield rcoro_scheduler.schedule(rcoro_save_proc)
assert isinstance(rcoro_save, asyncoro.Coro)
# if data is sent frequently (say, many times a second), enable
# streaming data to remote peer; this is more efficient as
# connections are kept open (so the cost of opening and closing
# connections is avoided), but keeping too many connections open
# consumes system resources
yield asyncoro.AsynCoro.instance().peer(rcoro_avg.location,
stream_send=True)
yield asyncoro.AsynCoro.instance().peer(rcoro_save.location,
stream_send=True)
# send 1000 items of random data to remote coroutines
for i in range(1000):
n = random.uniform(-1, 1)
item = (i, n)
# data can be sent to remote coroutines either with 'send' or
# 'deliver'; 'send' is more efficient but no guarantee data
# has been sent successfully whereas 'deliver' indicates
# errors right away; alternately, messages can be sent with a
# channel, which is more convenient if there are multiple
# (unknown) recipients
rcoro_avg.send(item)
rcoro_save.send(item)
yield coro.sleep(0.01)
item = (i, None)
rcoro_avg.send(item)
rcoro_save.send(item)
yield rcoro_scheduler.finish(close=True)
def setup_node(self, msg, coro=None):
if self._remote_scheduler:
yield self.asyncoro.peer(msg.info.location)
try:
params = yield asyncoro.Coro(
self._node_available, msg.info).finish()
except:
raise StopIteration
if not isinstance(params, tuple):
if hasattr(params, '__iter__'):
params = tuple(params)
else:
params = (params, )
msg = {
'req': 'setup_node',
'addr': msg.info.location.addr,
'params': params,
'auth': self.computation._auth,
'client': coro
}
self.computation.scheduler.send(msg)
def client_proc(computation, coro=None):
# scheduler sends node / server status messages to status_coro
computation.status_coro = asyncoro.Coro(status_proc)
# since RemoteCoroScheduler is not used, computation must be first scheduled
if (yield computation.schedule()):
raise Exception('schedule failed')
i = 0
while True:
cmd = yield coro.receive()
if cmd is None:
break
i += 1
c = C(i)
c.n = random.uniform(20, 50)
if cmd == 'servers':
yield computation.run_servers(compute, c, coro)
elif cmd == 'nodes':
yield computation.run_nodes(compute, c, coro)
else:
yield computation.run(compute, c, coro)
yield computation.close()
# client program for sending requests to remote server (tut_server.py)
# using message passing (asynchronous concurrent programming);
# see http://asyncoro.sourceforge.net/tutorial.html for details.
import sys, random
import asyncoro.disasyncoro as asyncoro
def client_proc(n, coro=None):
global msg_id
server = yield asyncoro.Coro.locate('server_coro')
for x in range(3):
# yield coro.suspend(random.uniform(0.5, 3))
msg_id += 1
server.send('%d: %d / %d' % (msg_id, n, x))
msg_id = 0
asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)
scheduler = asyncoro.AsynCoro(udp_port=0)
# create 10 clients; each client sends 3 messages
for i in range(10):
asyncoro.Coro(client_proc, i)
# in this case), the message is accepted by another scheduler
# (_ReactAsynCoro_) at the receiver and put in recipient's message
# queue
rcoro.send(random.uniform(10, 20))
# assume delay in input availability
yield coro.sleep(random.uniform(2, 5))
# end of input is indicated with None
rcoro.send(None)
result = yield coro.receive() # get result
print(' %s computed result: %.4f' % (rcoro.location, result))
for i in range(njobs):
rcoro = yield rcoro_scheduler.schedule(compute_coro)
if isinstance(rcoro, asyncoro.Coro):
print(' job %d processed by %s' % (i, rcoro.location))
asyncoro.Coro(send_requests, rcoro)
yield rcoro_scheduler.finish(close=True)
if __name__ == '__main__':
import random, sys
# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)
# if scheduler is not already running (on a node as a program), start
# private scheduler:
Scheduler()
# package computation fragments
computation = Computation([compute_coro])
# run n jobs
asyncoro.Coro(client_proc, computation, 10 if len(sys.argv) < 2 else int(sys.argv[1]))
def submit_jobs_proc(computation, njobs, coro=None):
# use RemoteCoroScheduler to schedule/submit coroutines; scheduler must be
# created before computation is scheduled (next step below)
rcoro_scheduler = RemoteCoroScheduler(computation)
for i in range(njobs):
# create remote coroutine
rcoro = yield rcoro_scheduler.schedule(rcoro_proc)
if isinstance(rcoro, asyncoro.Coro):
# create local coroutine to send input file and data to rcoro
asyncoro.Coro(client_proc, i, rcoro)
yield rcoro_scheduler.finish(close=True)
if __name__ == '__main__':
import random, os
# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)
# if scheduler is not already running (on a node as a program),
# start it (private scheduler):
Scheduler()
# unlike in earlier examples, rcoro_proc is not sent with
# computation (as it is not included in 'components';
# instead, it is sent each time a job is submitted,
# which is a bit inefficient
computation = Computation([C])
# run 10 jobs
asyncoro.Coro(submit_jobs_proc, computation, 10)
# send 1000 items of random data to remote coroutines
for i in range(1000):
n = random.uniform(-1, 1)
item = (i, n)
# data can be sent to remote coroutines either with 'send' or
# 'deliver'; 'send' is more efficient but no guarantee data
# has been sent successfully whereas 'deliver' indicates
# errors right away; alternately, messages can be sent with a
# channel, which is more convenient if there are multiple
# (unknown) recipients
rcoro_avg.send(item)
rcoro_save.send(item)
yield coro.sleep(0.01)
item = (i, None)
rcoro_avg.send(item)
rcoro_save.send(item)
yield computation.close()
if __name__ == '__main__':
import random
# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)
# if scheduler is shared (i.e., running as program), nothing needs
# to be done (its location can optionally be given to 'schedule');
# othrwise, start private scheduler:
Scheduler()
computation = Computation([])
asyncoro.Coro(client_proc, computation)
cmd, who = yield coro.receive()
# join/quit messages can be sent by clients themselves, but
# for illustration server sends them instead
if cmd == 'join':
channel.send(('joined', client_id))
who.send(client_id)
client_id += 1
elif cmd == 'quit':
channel.send(('bye', who))
elif cmd == 'terminate':
break
channel.unregister()
coro.unregister()
if __name__ == '__main__':
# asyncoro.logger.setLevel(asyncoro.Logger.DEBUG)
server = asyncoro.Coro(server_proc)
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
cmd = read_input()
if cmd.strip().lower() in ('quit', 'exit'):
break
except:
break
server.send(('terminate', None))
server.value() # wait for server to finish