def client_proc(job_id, data_file, rtask, task=None): # send input file to rtask.location; this will be saved to dispycos process's # working directory if (yield pycos.Pycos().send_file(rtask.location, data_file, timeout=10)) < 0: print('Could not send input data to %s' % rtask.location) # terminate remote task rtask.send(None) raise StopIteration(-1) # send info about input obj = C(job_id, data_file, random.uniform(5, 8), task) if (yield rtask.deliver(obj)) != 1: print('Could not send input to %s' % rtask.location) raise StopIteration(-1) # rtask sends result to this task as message result = yield task.receive() if not result.result_file: print('Processing %s failed' % obj.i) raise StopIteration(-1) # rtask saves results file at this client, which is saved in pycos's # dest_path, not current working directory! result_file = os.path.join(pycos.Pycos().dest_path, result.result_file) # move file to cwd target = os.path.join(os.getcwd(), os.path.basename(result_file)) os.rename(result_file, target) print(' job %s output is in %s' % (obj.i, target))
def rtask_proc(task=None): import os # receive object from client_proc task cobj = yield task.receive() if not cobj: raise StopIteration # Input file is already copied at where this rtask is running (by client). # For given input file, create an output file with each line in the output # file computed as length of corresponding line in input file cobj.result_file = 'result-%s' % cobj.data_file with open(cobj.data_file, 'r') as data_fd: with open(cobj.result_file, 'w') as result_fd: for lineno, line in enumerate(data_fd, start=1): result_fd.write('%d: %d\n' % (lineno, len(line) - 1)) # 'sleep' to simulate computing yield task.sleep(cobj.n) # transfer the result file to client status = yield pycos.Pycos().send_file(cobj.client.location, cobj.result_file, overwrite=True, timeout=30) if status: print('Could not send %s to %s' % (cobj.result_file, cobj.client.location)) cobj.result_file = None cobj.client.send(cobj) os.remove(cobj.data_file) os.remove(cobj.result_file)
def shutdown(self, wait=True): """This method should be called by user program to close the http server. If 'wait' is True the server waits for poll_sec so the http client gets all the updates before server is closed. """ if wait: pycos.logger.info( 'HTTP server waiting for %s seconds for client updates ' 'before quitting', self._poll_sec) if pycos.Pycos().cur_task(): def _shutdown(task=None): yield task.sleep(self._poll_sec + 0.5) self._server.shutdown() self._server.server_close() pycos.SysTask(_shutdown) else: time.sleep(self._poll_sec + 0.5) self._server.shutdown() self._server.server_close() else: self._server.shutdown() self._server.server_close()
def server_available(location, data_file, task=None): # 'server_available' is executed locally (at client) when a server process # is available. 'location' is Location instance of server. When this task is # executed, 'depends' of computation would've been transferred. data_file # could've been sent with the computation 'depends'; however, to illustrate # how files can be sent separately (to distribute data fragments among # servers), files are transferred to servers in this example print(' Sending %s to %s' % (data_file, location)) if (yield pycos.Pycos().send_file( location, data_file, timeout=5, overwrite=True)) < 0: print('Could not send data file "%s" to %s' % (data_file, location)) raise StopIteration(-1) # 'setup_server' is executed on remote server at 'location' with argument # data_file yield computation.enable_server(location, data_file) raise StopIteration(0)
def node_available(avail_info, data_file, task=None): # 'node_available' is executed locally (at client) when a node is # available. 'location' is Location instance of node. When this task is # executed, 'depends' of computation would've been transferred. # data_file could've been sent with the computation 'depends'; however, to # illustrate how files can be sent separately (e.g., to transfer different # files to different nodes), file is transferred with 'node_available'. print(' Sending %s to %s' % (data_file, avail_info.location.addr)) sent = yield pycos.Pycos().send_file(avail_info.location, data_file, overwrite=True, timeout=5) if (sent < 0): print('Could not send data file "%s" to %s' % (data_file, avail_info.location)) raise StopIteration(-1) # value of task (last value yield'ed or value of 'raise StopIteration') will # be passed to node_setup as argument(s). yield computation.enable_node(avail_info.location.addr, data_file)
def client_proc(computation, njobs, task=None): # schedule computation with the scheduler; scheduler accepts one computation # at a time, so if scheduler is shared, the computation is queued until it # is done with already scheduled computations if (yield computation.schedule()): raise Exception('Could not schedule computation') # pair EC2 node with this client with: yield pycos.Pycos().peer(pycos.Location('54.204.242.185', 9706)) # if multiple nodes are used, 'relay' option can be used to pair with # all nodes with just one statement as: # yield pycos.Pycos().peer(pycos.Location('54.204.242.185', 9706), relay=True) # execute n jobs (tasks) and get their results. Note that number of # jobs created can be more than number of server processes available; the # scheduler will use as many processes as necessary/available, running one # job at a server process args = [(i, random.uniform(3, 10)) for i in range(njobs)] results = yield computation.run_results(compute, args) for result in results: print('job %s result: %s' % (result[0], result[1])) yield computation.close()
yield client.close() if __name__ == '__main__': import sys, random import pycos import pycos.netpycos from pycos.dispycos import * # enable debug to see progress pycos.logger.setLevel(pycos.Logger.DEBUG) # PyPI / pip packaging adjusts assertion below for Python 3.7+ if sys.version_info.major == 3: assert sys.version_info.minor < 7, \ ('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' % (__file__, sys.version_info.major, sys.version_info.minor)) config = {} # add any additional parameters # if client is behind a router, configure router's firewall to forward port # 9705 to client's IP address and use router's external IP address (i.e., # addressable from outside world) config['ext_ip_addr'] = 'router.ext.ip' pycos.Pycos(**config) # send 'compute' to dispycos servers to run tasks when jobs are scheduled client = Client([compute]) njobs = 4 if len(sys.argv) == 1 else int(sys.argv[1]) pycos.Task(client_proc, njobs)
print(' ** run failed for %s' % i) # wait for results for rtask in rtasks: result = yield rtask() if isinstance(result, tuple) and len(result) == 2: print(' result from %s: %s' % (rtask.location, str(result))) else: print(' ** rtask %s failed' % rtask) yield client.close() if __name__ == '__main__': import sys, random import pycos import pycos.netpycos from pycos.dispycos import * # pycos.logger.setLevel(pycos.Logger.DEBUG) # PyPI / pip packaging adjusts assertion below for Python 3.7+ if sys.version_info.major == 3: assert sys.version_info.minor < 7, \ ('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' % (__file__, sys.version_info.major, sys.version_info.minor)) pycos.Pycos(host='127.0.0.1', tcp_port=9705, udp_port=9705) # send 'compute' to remote servers to run tasks when submitted client = Client([compute]) njobs = 10 if len(sys.argv) == 1 else int(sys.argv[1]) pycos.Task(client_proc, njobs)
server = yield rps(client) if isinstance(server, pycos.Task): servers[location] = server pycos.Task(discover_rps, status.location) else: # status.status == pycos.PeerStatus.Offline servers.pop(status.location, None) for rps in rpss.values(): rps.close() if __name__ == '__main__': pycos.logger.setLevel(pycos.Logger.DEBUG) # use 'secret' as used by server scheduler = pycos.Pycos(name='client', secret='LogMon') servers = {} # set peer_status notification status_monitor = pycos.Task(peer_status) scheduler.peer_status(status_monitor) if sys.version_info.major > 2: read_input = input else: read_input = raw_input while True: try: line = read_input( 'Enter "quit" or "exit" to quit: ').strip().lower() if line in ('quit', 'exit'): break
# to illustrate how client's monitor can receive exit values or exceptions, exception is # raised if given b is not a positve number, otherwise task sleeps for b seconds and exits # with msg if isinstance(b, (int, float)) and b > 0 and isinstance(msg, str): yield task.sleep(b) # (remote) monitor (if any) gets back msg (to be interpreted as normal termination) raise StopIteration(msg) else: # (remote) monitor (if any) gets this exception raise Exception('invalid invocation: %s' % b) if __name__ == '__main__': pycos.logger.setLevel(pycos.Logger.DEBUG) # 'secret' is set so only peers that use same secret can communicate scheduler = pycos.Pycos(name='server', secret='test') # register rps_server so remote clients can request execution rps = pycos.RPS(rps_server) rps.register() if sys.version_info.major > 2: read_input = input else: read_input = raw_input while True: try: line = read_input().strip().lower() if line in ('quit', 'exit'): break except Exception: break
# execute n jobs (tasks) and get their results. Note that number of # jobs created can be more than number of server processes available; the # scheduler will use as many processes as necessary/available, running one # job at a server process args = [(i, random.uniform(3, 10)) for i in range(njobs)] results = yield computation.run_results(compute, args) for result in results: print('job %s result: %s' % (result[0], result[1])) yield computation.close() if __name__ == '__main__': import sys, random # pycos.logger.setLevel(pycos.Logger.DEBUG) # PyPI / pip packaging adjusts assertion below for Python 3.7+ if sys.version_info.major == 3: assert sys.version_info.minor < 7, \ ('"%s" is not suitable for Python version %s.%s; use file installed by pip instead' % (__file__, sys.version_info.major, sys.version_info.minor)) pycos.Pycos(node='127.0.0.1', tcp_port=9705, udp_port=9705) njobs = 10 if len(sys.argv) == 1 else int(sys.argv[1]) # if scheduler is not already running (on a node as a program), # start private scheduler: Scheduler() # use 'compute' for computation jobs computation = Computation([compute]) pycos.Task(client_proc, computation, njobs)
else: pycos.logger.warning('ignoring invalid message') def rti_test(task=None): # if server is on remote network, automatic discovery won't work, # so add it explicitly # yield scheduler.peer(pycos.Location('192.168.21.5', 9705)) # get reference to RTI at server rti1 = yield pycos.RTI.locate('rti_1') print('RTI is at %s' % rti1.location) # 5 (remote) tasks are created with rti1 n = 5 # set monitor (monitor_proc task) for tasks created for this RTI yield rti1.monitor(pycos.Task(monitor_proc, n)) for i in range(n): rtask = yield rti1('test%s' % i, b=i) pycos.logger.debug('RTI %s created' % rtask) # If necessary, each rtask can also be set (different) 'monitor' rtask.send('msg:%s' % i) yield task.sleep(random.uniform(0, 1)) pycos.logger.setLevel(pycos.Logger.DEBUG) # use 'test' secret so peers that use same secret are recognized scheduler = pycos.Pycos(name='client', secret='test') pycos.Task(rti_test)
if req.get('name', None) != 'time': raise Exception('request should have "name" set to "time"') if not isinstance(client, pycos.Task): raise Exception( 'request should have "client" set to task of requester') delay = random.uniform(0.5, 2) # simulate delay in getting result (e.g., reading a sensor or computation) yield task.sleep(delay) raise StopIteration({'result': time.asctime(), 'server': task}) if __name__ == '__main__': # pycos.logger.setLevel(pycos.Logger.DEBUG) # 'secret' is set so only peers that use same secret can communicate; # SSL can be used for encryption if required; see 'rps_node_*.py' for authentication of peers scheduler = pycos.Pycos(name='pico_server', secret='PycOS') # create RPS and register it so remote clients can request execution pycos.RPS(pico_service).register() if sys.version_info.major > 2: read_input = input else: read_input = raw_input while 1: try: line = read_input( 'Enter "quit" or "exit" to terminate pico_service: ').strip( ).lower() if line in ('quit', 'exit'): break
for i in range(n): req = {'name': 'time', 'client': task} rtask = yield service_rps(req) if isinstance(rtask, pycos.Task): pycos.logger.info('service task: %s', rtask) service_tasks.append(rtask) for rtask in service_tasks: reply = yield rtask.finish(timeout=5) # use timeout in case of failures if isinstance(reply, dict): # assert rtask == reply.get('server') pycos.logger.info('result: %s, from: %s', reply.get('result', None), reply.get('server', None)) elif isinstance(reply, pycos.MonitorStatus): # this warning is also shown by RPS monitor; in real use cases this error may be # handled differently pycos.logger.warning('rtask %s failed: %s with %s', rtask, reply.type, reply.value) else: pycos.logger.warning('invalid reply: %s', type(reply)) if __name__ == '__main__': # pycos.logger.setLevel(pycos.Logger.DEBUG) if len(sys.argv) > 1: n = int(sys.argv[1]) else: n = 5 # use 'PycOS' secret so peers that use same secret are recognized scheduler = pycos.Pycos(secret='PycOS') pycos.Task(client_proc, n)
if __name__ == '__main__': pycos.logger.setLevel(pycos.Logger.DEBUG) # file to monitor - can be changed with first argument to this program log_file = '/var/log/auth.log' # 'grep' regexp pattern to filter - can be changed with second argument to this program pattern = 'failed' if len(sys.argv) > 1: log_file = sys.argv[1] if len(sys.argv) > 2: pattern = sys.argv[2] # use 'secret' so peers that use same secret are recognized scheduler = pycos.Pycos(name='server', secret='LogMon') # register rps_server so remote clients can request execution rps = pycos.RPS(rps_log_monitor) rps.register() if sys.version_info.major > 2: read_input = input else: read_input = raw_input while True: try: line = read_input( 'Enter "quit" or "exit" to quit: ').strip().lower() if line in ('quit', 'exit'): break except Exception: