def setup(self, peer, timeout=None, func=None, args=(), kwargs={}): """Sends computation to peer. This method should be called once for each peer. Gives 0 if the setup is successful. Must be used with yield as 'v = yield computation.setup(peer)' """ if not self._code: raise StopIteration(-1) def _setup(self, peer, timeout, func, name, args, kwargs, coro=None): peer.send({ 'cmd': 'setup', 'client': coro, 'computation': self, 'func': func, 'name': name, 'args': args, 'kwargs': kwargs }) reply = yield coro.receive(timeout=timeout) if reply != 0: raise StopIteration(-1) for i, xf in enumerate(self._xfer_files): reply = yield Computation._asyncoro.send_file(peer.location, xf, timeout=timeout) if reply < 0: asyncoro.logger.debug('failed to transfer file %s: %s' % (xf, reply)) self._xfer_files = self._xfer_files[:i] yield self.close(peer) raise StopIteration(-1) raise StopIteration(0) name = None if func: if inspect.isfunction(func): if sys.version_info.major >= 3: name = func.__name__ else: name = func.func_name func = inspect.getsource(func).lstrip() else: logger.warning('Invalid setup function ignored') func = None args = () kwargs = {} coro = asyncoro.Coro(_setup, self, peer, timeout, func, name, args, kwargs) yield coro.finish()
def run(self, peer, *args, **kwargs): """Once 'setup', the computation can be run as many times as necessary. Each 'run' creates a coroutine at peer and returns reference to that remote coroutine if successful; otherwise, gives -1. Must be used with yield as 'rcoro = yield computation.run(peer, ...)' """ def _run(self, peer, coro=None): peer.send({ 'cmd': 'run', 'client': coro, 'computation': self._name, 'args': args, 'kwargs': kwargs }) yield coro.receive(timeout=5) coro = asyncoro.Coro(_run, self, peer) yield coro.finish()
def send_proc(coro=None): # if server is in a remote network, use 'peer' as (optionally # enabling streaming for efficiency): # yield asyncoro.AsynCoro.instance().peer('server node/ip') server = yield asyncoro.Coro.locate('server') yield server.send(('join', coro)) client_id = yield coro.receive() channel = yield asyncoro.Channel.locate('channel') recv_coro = asyncoro.Coro(recv_proc, client_id) yield channel.subscribe(recv_coro) # since readline is synchronous (blocking) call, use async thread async_threads = asyncoro.AsyncThreadPool(1) while True: msg = yield async_threads.async_task(sys.stdin.readline) msg = msg.strip() if msg == 'quit': break channel.send((msg, client_id)) server.send(('quit', client_id)) yield channel.unsubscribe(recv_coro)
continue print(' %s %s' % (who, msg)) def send_proc(coro=None): # if server is in a remote network, use 'peer' as (optionally # enabling streaming for efficiency): # yield asyncoro.AsynCoro.instance().peer('server node/ip') server = yield asyncoro.Coro.locate('server') yield server.send(('join', coro)) client_id = yield coro.receive() channel = yield asyncoro.Channel.locate('channel') recv_coro = asyncoro.Coro(recv_proc, client_id) yield channel.subscribe(recv_coro) # since readline is synchronous (blocking) call, use async thread async_threads = asyncoro.AsyncThreadPool(1) while True: msg = yield async_threads.async_task(sys.stdin.readline) msg = msg.strip() if msg == 'quit': break channel.send((msg, client_id)) server.send(('quit', client_id)) yield channel.unsubscribe(recv_coro) if __name__ == '__main__': # asyncoro.logger.setLevel(logging.DEBUG) asyncoro.Coro(send_proc)
def _discoro_process(_discoro_config, _discoro_queue): _discoro_scheduler = asyncoro.AsynCoro(**_discoro_config) _discoro_coro = asyncoro.Coro(discoro_server) _discoro_queue.put(asyncoro.serialize(_discoro_coro)) _discoro_scheduler.finish()
def discoro_server(_discoro_name='discoro_server'): """Server that receives computations and runs coroutines for it. """ _discoro_coro = asyncoro.AsynCoro.cur_coro() _discoro_coro.register(_discoro_name) asyncoro.logger.debug('discoro %s started' % _discoro_coro) # os.chdir(asyncoro.AsynCoro.instance().dest_path) _discoro_computations = {} _discoro_globals = _discoro_locals = _discoro_cmd = _discoro_client = None _discoro_msg = _discoro_computation = _discoro_job_coro = None _discoro_job_coros = set() _discoro_globals = list(globals().keys()) _discoro_locals = list(locals().keys()) _discoro_globals.extend(_discoro_locals) while True: _discoro_msg = yield _discoro_coro.receive() try: _discoro_cmd = _discoro_msg['cmd'] except: if isinstance(_discoro_msg, asyncoro.MonitorException): asyncoro.logger.debug('job %s done' % _discoro_msg.args[0]) _discoro_job_coros.discard(_discoro_msg.args[0]) continue else: asyncoro.logger.debug(traceback.format_exc()) _discoro_cmd = None if _discoro_cmd == 'run': _discoro_client = _discoro_msg.get('client', None) if not isinstance(_discoro_client, asyncoro.Coro): continue try: _discoro_computation = _discoro_computations[ _discoro_msg['computation']] job_coro = asyncoro.Coro(globals()[_discoro_computation._name], *_discoro_msg['args'], **_discoro_msg['kwargs']) except: asyncoro.logger.debug('invalid computation to run') asyncoro.logger.debug(traceback.format_exc()) job_coro = -1 else: asyncoro.logger.debug('job %s created' % job_coro) _discoro_job_coros.add(job_coro) if (yield _discoro_coro.monitor(job_coro)) != 0: _discoro_job_coros.discard(job_coro) _discoro_client.send(job_coro) del job_coro elif _discoro_cmd == 'ping': # TODO: maintain heartbeat times and remove computations from dead clients? pass elif _discoro_cmd == 'setup': _discoro_client = _discoro_msg.get('client', None) if not isinstance(_discoro_client, asyncoro.Coro): continue try: _discoro_computation = _discoro_msg['computation'] exec(compile(_discoro_computation._code, '<string>', 'exec')) globals().update(locals()) if _discoro_msg['func']: try: exec(compile(_discoro_msg['func'], '<string>', 'exec')) locals()[_discoro_msg['name']]( *_discoro_msg['args'], **_discoro_msg['kwargs']) except: logger.warning('setup function failed: %s' % traceback.format_exc()) except: asyncoro.logger.warning('invalid computation') asyncoro.logger.debug(traceback.format_exc()) _discoro_client.send(-1) continue asyncoro.logger.debug('computation "%s" from "%s"' % \ (_discoro_computation._name, _discoro_msg['client'])) setattr(_discoro_computation, '_client', _discoro_client) setattr(_discoro_computation, '_globals', [_discoro_var for _discoro_var in globals().keys() \ if _discoro_var not in _discoro_globals]) setattr(_discoro_computation, '_locals', [_discoro_var for _discoro_var in locals().keys() \ if _discoro_var not in _discoro_locals]) _discoro_globals.extend(_discoro_computation._globals) _discoro_locals.extend(_discoro_computation._locals) _discoro_computations[ _discoro_computation._name] = _discoro_computation _discoro_client.send(0) elif _discoro_cmd == 'close': try: _discoro_computation = _discoro_computations.pop( _discoro_msg['computation'], None) asyncoro.logger.debug('deleting computation "%s"' % _discoro_computation._name) if _discoro_msg['func']: try: exec(compile(_discoro_msg['func'], '<string>', 'exec')) locals()[_discoro_msg['name']]( *_discoro_msg['args'], **_discoro_msg['kwargs']) except: logger.warning('cleanup function failed: %s' % traceback.format_exc()) for _discoro_var in _discoro_computation._globals: globals().pop(_discoro_var, None) for _discoro_var in _discoro_computation._locals: locals().pop(_discoro_var, None) except: asyncoro.logger.warning('invalid computation "%s" to delete' % \ _discoro_msg.get('computation', None)) elif _discoro_cmd == 'quit': break else: asyncoro.logger.warning('invalid command "%s" ignored' % _discoro_cmd) _discoro_client = _discoro_msg.get('client', None) if not isinstance(_discoro_client, asyncoro.Coro): continue _discoro_client.send(-1) # wait until all computations are done; process only 'ping' and 'close' while _discoro_job_coros: _discoro_msg = yield _discoro_coro.receive() try: _discoro_cmd = _discoro_msg['cmd'] except: if isinstance(_discoro_msg, asyncoro.MonitorException): asyncoro.logger.debug('job %s done' % _discoro_msg.args[0]) _discoro_job_coros.discard(_discoro_msg.args[0]) continue else: asyncoro.logger.debug(traceback.format_exc()) _discoro_cmd = None if _discoro_cmd == 'ping': # TODO: maintain heartbeat times and remove computations from dead clients? pass elif _discoro_cmd == 'close': try: _discoro_computation = _discoro_computations.pop( _discoro_msg['computation'], None) asyncoro.logger.debug('deleting computation "%s"' % _discoro_computation._name) for _discoro_var in _discoro_computation._globals: globals().pop(_discoro_var, None) for _discoro_var in _discoro_computation._locals: locals().pop(_discoro_var, None) if not _discoro_computations: break except: asyncoro.logger.warning('invalid computation to delete') else: asyncoro.logger.warning('invalid command "%s" ignored' % _discoro_cmd) _discoro_client = _discoro_msg.get('client', None) if not isinstance(_discoro_client, asyncoro.Coro): continue _discoro_client.send(-1) for _discoro_computation in _discoro_computations.values(): asyncoro.logger.debug('deleting computation "%s"' % _discoro_computation._name) for _discoro_var in _discoro_computation._globals: globals().pop(_discoro_var, None) for _discoro_var in _discoro_computation._locals: locals().pop(_discoro_var, None) for _discoro_var in _discoro_locals: if _discoro_var == '_discoro_locals': continue globals().pop(_discoro_var, None) globals().pop('_discoro_locals', None)
_discoro_name = _discoro_config['name'] _discoro_queue = multiprocessing.Queue() _discoro_processes = [] for _discoro_proc_id in range(2, _discoro_cpus + 1): _discoro_config['name'] = _discoro_name + '-%s' % _discoro_proc_id _discoro_processes.append( multiprocessing.Process(target=_discoro_process, args=(_discoro_config, _discoro_queue))) _discoro_processes[-1].start() time.sleep(0.05) _discoro_proc_id = 1 _discoro_config['name'] = _discoro_name + '-%s' % _discoro_proc_id _discoro_scheduler = asyncoro.AsynCoro(**_discoro_config) _discoro_coro = asyncoro.Coro(discoro_server) while True: try: if sys.stdin.readline().strip().lower() in ('quit', 'exit'): break except KeyboardInterrupt: break asyncoro.logger.debug('terminating servers') while not _discoro_queue.empty(): asyncoro.unserialize(_discoro_queue.get()).send({'cmd': 'quit'}) _discoro_coro.send({'cmd': 'quit'}) for _i, _discoro_proc in enumerate(_discoro_processes, start=1): if _discoro_proc.is_alive():
coro.register('server') client_id = 1 while True: cmd, who = yield coro.receive() # join/quit messages can be sent by clients themselves, but # for illustration server sends them instead if cmd == 'join': channel.send(('joined', client_id)) who.send(client_id) client_id += 1 elif cmd == 'quit': channel.send(('bye', who)) elif cmd == 'terminate': break channel.unregister() coro.unregister() if __name__ == '__main__': # asyncoro.logger.setLevel(logging.DEBUG) server = asyncoro.Coro(server_proc) while True: try: cmd = sys.stdin.readline().strip() if cmd == 'quit': break except KeyboardInterrupt: break server.send(('terminate', None)) server.value() # wait for server to finish