def client_proc(job_id, data_file, rtask, task=None):
# send input file to rtask.location; this will be saved to dispycos process's
# working directory
if (yield pycos.Pycos().send_file(rtask.location, data_file,
timeout=10)) < 0:
print('Could not send input data to %s' % rtask.location)
# terminate remote task
rtask.send(None)
raise StopIteration(-1)
# send info about input
obj = C(job_id, data_file, random.uniform(5, 8), task)
if (yield rtask.deliver(obj)) != 1:
print('Could not send input to %s' % rtask.location)
raise StopIteration(-1)
# rtask sends result to this task as message
result = yield task.receive()
if not result.result_file:
print('Processing %s failed' % obj.i)
raise StopIteration(-1)
# rtask saves results file at this client, which is saved in pycos's
# dest_path, not current working directory!
result_file = os.path.join(pycos.Pycos().dest_path, result.result_file)
# move file to cwd
target = os.path.join(os.getcwd(), os.path.basename(result_file))
os.rename(result_file, target)
print(' job %s output is in %s' % (obj.i, target))
def rtask_proc(task=None):
import os
# receive object from client_proc task
cobj = yield task.receive()
if not cobj:
raise StopIteration
# Input file is already copied at where this rtask is running (by client).
# For given input file, create an output file with each line in the output
# file computed as length of corresponding line in input file
cobj.result_file = 'result-%s' % cobj.data_file
with open(cobj.data_file, 'r') as data_fd:
with open(cobj.result_file, 'w') as result_fd:
for lineno, line in enumerate(data_fd, start=1):
result_fd.write('%d: %d\n' % (lineno, len(line) - 1))
# 'sleep' to simulate computing
yield task.sleep(cobj.n)
# transfer the result file to client
status = yield pycos.Pycos().send_file(cobj.client.location,
cobj.result_file,
overwrite=True,
timeout=30)
if status:
print('Could not send %s to %s' %
(cobj.result_file, cobj.client.location))
cobj.result_file = None
cobj.client.send(cobj)
os.remove(cobj.data_file)
os.remove(cobj.result_file)
def shutdown(self, wait=True):
"""This method should be called by user program to close the
http server. If 'wait' is True the server waits for poll_sec
so the http client gets all the updates before server is
closed.
"""
if wait:
pycos.logger.info(
'HTTP server waiting for %s seconds for client updates '
'before quitting', self._poll_sec)
if pycos.Pycos().cur_task():
def _shutdown(task=None):
yield task.sleep(self._poll_sec + 0.5)
self._server.shutdown()
self._server.server_close()
pycos.SysTask(_shutdown)
else:
time.sleep(self._poll_sec + 0.5)
self._server.shutdown()
self._server.server_close()
else:
self._server.shutdown()
self._server.server_close()
def server_available(location, data_file, task=None):
# 'server_available' is executed locally (at client) when a server process
# is available. 'location' is Location instance of server. When this task is
# executed, 'depends' of computation would've been transferred. data_file
# could've been sent with the computation 'depends'; however, to illustrate
# how files can be sent separately (to distribute data fragments among
# servers), files are transferred to servers in this example
print(' Sending %s to %s' % (data_file, location))
if (yield pycos.Pycos().send_file(
location, data_file, timeout=5, overwrite=True)) < 0:
print('Could not send data file "%s" to %s' % (data_file, location))
raise StopIteration(-1)
# 'setup_server' is executed on remote server at 'location' with argument
# data_file
yield computation.enable_server(location, data_file)
raise StopIteration(0)
def node_available(avail_info, data_file, task=None):
# 'node_available' is executed locally (at client) when a node is
# available. 'location' is Location instance of node. When this task is
# executed, 'depends' of computation would've been transferred.
# data_file could've been sent with the computation 'depends'; however, to
# illustrate how files can be sent separately (e.g., to transfer different
# files to different nodes), file is transferred with 'node_available'.
print(' Sending %s to %s' % (data_file, avail_info.location.addr))
sent = yield pycos.Pycos().send_file(avail_info.location,
data_file,
overwrite=True,
timeout=5)
if (sent < 0):
print('Could not send data file "%s" to %s' %
(data_file, avail_info.location))
raise StopIteration(-1)
# value of task (last value yield'ed or value of 'raise StopIteration') will
# be passed to node_setup as argument(s).
yield computation.enable_node(avail_info.location.addr, data_file)
def client_proc(computation, njobs, task=None):
# schedule computation with the scheduler; scheduler accepts one computation
# at a time, so if scheduler is shared, the computation is queued until it
# is done with already scheduled computations
if (yield computation.schedule()):
raise Exception('Could not schedule computation')
# pair EC2 node with this client with:
yield pycos.Pycos().peer(pycos.Location('54.204.242.185', 9706))
# if multiple nodes are used, 'relay' option can be used to pair with
# all nodes with just one statement as:
# yield pycos.Pycos().peer(pycos.Location('54.204.242.185', 9706), relay=True)
# execute n jobs (tasks) and get their results. Note that number of
# jobs created can be more than number of server processes available; the
# scheduler will use as many processes as necessary/available, running one
# job at a server process
args = [(i, random.uniform(3, 10)) for i in range(njobs)]
results = yield computation.run_results(compute, args)
for result in results:
print('job %s result: %s' % (result[0], result[1]))
yield computation.close()
yield client.close()
if __name__ == '__main__':
import sys, random
import pycos
import pycos.netpycos
from pycos.dispycos import *
# enable debug to see progress
pycos.logger.setLevel(pycos.Logger.DEBUG)
# PyPI / pip packaging adjusts assertion below for Python 3.7+
if sys.version_info.major == 3:
assert sys.version_info.minor < 7, \
('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' %
(__file__, sys.version_info.major, sys.version_info.minor))
config = {} # add any additional parameters
# if client is behind a router, configure router's firewall to forward port
# 9705 to client's IP address and use router's external IP address (i.e.,
# addressable from outside world)
config['ext_ip_addr'] = 'router.ext.ip'
pycos.Pycos(**config)
# send 'compute' to dispycos servers to run tasks when jobs are scheduled
client = Client([compute])
njobs = 4 if len(sys.argv) == 1 else int(sys.argv[1])
pycos.Task(client_proc, njobs)
print(' ** run failed for %s' % i)
# wait for results
for rtask in rtasks:
result = yield rtask()
if isinstance(result, tuple) and len(result) == 2:
print(' result from %s: %s' % (rtask.location, str(result)))
else:
print(' ** rtask %s failed' % rtask)
yield client.close()
if __name__ == '__main__':
import sys, random
import pycos
import pycos.netpycos
from pycos.dispycos import *
# pycos.logger.setLevel(pycos.Logger.DEBUG)
# PyPI / pip packaging adjusts assertion below for Python 3.7+
if sys.version_info.major == 3:
assert sys.version_info.minor < 7, \
('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' %
(__file__, sys.version_info.major, sys.version_info.minor))
pycos.Pycos(host='127.0.0.1', tcp_port=9705, udp_port=9705)
# send 'compute' to remote servers to run tasks when submitted
client = Client([compute])
njobs = 10 if len(sys.argv) == 1 else int(sys.argv[1])
pycos.Task(client_proc, njobs)
server = yield rps(client)
if isinstance(server, pycos.Task):
servers[location] = server
pycos.Task(discover_rps, status.location)
else: # status.status == pycos.PeerStatus.Offline
servers.pop(status.location, None)
for rps in rpss.values():
rps.close()
if __name__ == '__main__':
pycos.logger.setLevel(pycos.Logger.DEBUG)
# use 'secret' as used by server
scheduler = pycos.Pycos(name='client', secret='LogMon')
servers = {}
# set peer_status notification
status_monitor = pycos.Task(peer_status)
scheduler.peer_status(status_monitor)
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
line = read_input(
'Enter "quit" or "exit" to quit: ').strip().lower()
if line in ('quit', 'exit'):
break
# to illustrate how client's monitor can receive exit values or exceptions, exception is
# raised if given b is not a positve number, otherwise task sleeps for b seconds and exits
# with msg
if isinstance(b, (int, float)) and b > 0 and isinstance(msg, str):
yield task.sleep(b)
# (remote) monitor (if any) gets back msg (to be interpreted as normal termination)
raise StopIteration(msg)
else:
# (remote) monitor (if any) gets this exception
raise Exception('invalid invocation: %s' % b)
if __name__ == '__main__':
pycos.logger.setLevel(pycos.Logger.DEBUG)
# 'secret' is set so only peers that use same secret can communicate
scheduler = pycos.Pycos(name='server', secret='test')
# register rps_server so remote clients can request execution
rps = pycos.RPS(rps_server)
rps.register()
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
line = read_input().strip().lower()
if line in ('quit', 'exit'):
break
except Exception:
break
# execute n jobs (tasks) and get their results. Note that number of
# jobs created can be more than number of server processes available; the
# scheduler will use as many processes as necessary/available, running one
# job at a server process
args = [(i, random.uniform(3, 10)) for i in range(njobs)]
results = yield computation.run_results(compute, args)
for result in results:
print('job %s result: %s' % (result[0], result[1]))
yield computation.close()
if __name__ == '__main__':
import sys, random
# pycos.logger.setLevel(pycos.Logger.DEBUG)
# PyPI / pip packaging adjusts assertion below for Python 3.7+
if sys.version_info.major == 3:
assert sys.version_info.minor < 7, \
('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' %
(__file__, sys.version_info.major, sys.version_info.minor))
pycos.Pycos(node='127.0.0.1', tcp_port=9705, udp_port=9705)
njobs = 10 if len(sys.argv) == 1 else int(sys.argv[1])
# if scheduler is not already running (on a node as a program),
# start private scheduler:
Scheduler()
# use 'compute' for computation jobs
computation = Computation([compute])
pycos.Task(client_proc, computation, njobs)
else:
pycos.logger.warning('ignoring invalid message')
def rti_test(task=None):
# if server is on remote network, automatic discovery won't work,
# so add it explicitly
# yield scheduler.peer(pycos.Location('192.168.21.5', 9705))
# get reference to RTI at server
rti1 = yield pycos.RTI.locate('rti_1')
print('RTI is at %s' % rti1.location)
# 5 (remote) tasks are created with rti1
n = 5
# set monitor (monitor_proc task) for tasks created for this RTI
yield rti1.monitor(pycos.Task(monitor_proc, n))
for i in range(n):
rtask = yield rti1('test%s' % i, b=i)
pycos.logger.debug('RTI %s created' % rtask)
# If necessary, each rtask can also be set (different) 'monitor'
rtask.send('msg:%s' % i)
yield task.sleep(random.uniform(0, 1))
pycos.logger.setLevel(pycos.Logger.DEBUG)
# use 'test' secret so peers that use same secret are recognized
scheduler = pycos.Pycos(name='client', secret='test')
pycos.Task(rti_test)
if req.get('name', None) != 'time':
raise Exception('request should have "name" set to "time"')
if not isinstance(client, pycos.Task):
raise Exception(
'request should have "client" set to task of requester')
delay = random.uniform(0.5, 2)
# simulate delay in getting result (e.g., reading a sensor or computation)
yield task.sleep(delay)
raise StopIteration({'result': time.asctime(), 'server': task})
if __name__ == '__main__':
# pycos.logger.setLevel(pycos.Logger.DEBUG)
# 'secret' is set so only peers that use same secret can communicate;
# SSL can be used for encryption if required; see 'rps_node_*.py' for authentication of peers
scheduler = pycos.Pycos(name='pico_server', secret='PycOS')
# create RPS and register it so remote clients can request execution
pycos.RPS(pico_service).register()
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while 1:
try:
line = read_input(
'Enter "quit" or "exit" to terminate pico_service: ').strip(
).lower()
if line in ('quit', 'exit'):
break
for i in range(n):
req = {'name': 'time', 'client': task}
rtask = yield service_rps(req)
if isinstance(rtask, pycos.Task):
pycos.logger.info('service task: %s', rtask)
service_tasks.append(rtask)
for rtask in service_tasks:
reply = yield rtask.finish(timeout=5) # use timeout in case of failures
if isinstance(reply, dict):
# assert rtask == reply.get('server')
pycos.logger.info('result: %s, from: %s',
reply.get('result', None), reply.get('server', None))
elif isinstance(reply, pycos.MonitorStatus):
# this warning is also shown by RPS monitor; in real use cases this error may be
# handled differently
pycos.logger.warning('rtask %s failed: %s with %s', rtask, reply.type, reply.value)
else:
pycos.logger.warning('invalid reply: %s', type(reply))
if __name__ == '__main__':
# pycos.logger.setLevel(pycos.Logger.DEBUG)
if len(sys.argv) > 1:
n = int(sys.argv[1])
else:
n = 5
# use 'PycOS' secret so peers that use same secret are recognized
scheduler = pycos.Pycos(secret='PycOS')
pycos.Task(client_proc, n)
if __name__ == '__main__':
pycos.logger.setLevel(pycos.Logger.DEBUG)
# file to monitor - can be changed with first argument to this program
log_file = '/var/log/auth.log'
# 'grep' regexp pattern to filter - can be changed with second argument to this program
pattern = 'failed'
if len(sys.argv) > 1:
log_file = sys.argv[1]
if len(sys.argv) > 2:
pattern = sys.argv[2]
# use 'secret' so peers that use same secret are recognized
scheduler = pycos.Pycos(name='server', secret='LogMon')
# register rps_server so remote clients can request execution
rps = pycos.RPS(rps_log_monitor)
rps.register()
if sys.version_info.major > 2:
read_input = input
else:
read_input = raw_input
while True:
try:
line = read_input(
'Enter "quit" or "exit" to quit: ').strip().lower()
if line in ('quit', 'exit'):
break
except Exception:
