コード例 #1
0
ファイル: discoro.py プロジェクト: yonglehou/asyncoro
    def setup(self, peer, timeout=None, func=None, args=(), kwargs={}):
        """Sends computation to peer. This method should be called
        once for each peer. Gives 0 if the setup is successful.

        Must be used with yield as 'v = yield computation.setup(peer)'
        """
        if not self._code:
            raise StopIteration(-1)

        def _setup(self, peer, timeout, func, name, args, kwargs, coro=None):
            peer.send({
                'cmd': 'setup',
                'client': coro,
                'computation': self,
                'func': func,
                'name': name,
                'args': args,
                'kwargs': kwargs
            })
            reply = yield coro.receive(timeout=timeout)
            if reply != 0:
                raise StopIteration(-1)
            for i, xf in enumerate(self._xfer_files):
                reply = yield Computation._asyncoro.send_file(peer.location,
                                                              xf,
                                                              timeout=timeout)
                if reply < 0:
                    asyncoro.logger.debug('failed to transfer file %s: %s' %
                                          (xf, reply))
                    self._xfer_files = self._xfer_files[:i]
                    yield self.close(peer)
                    raise StopIteration(-1)
            raise StopIteration(0)

        name = None
        if func:
            if inspect.isfunction(func):
                if sys.version_info.major >= 3:
                    name = func.__name__
                else:
                    name = func.func_name
                func = inspect.getsource(func).lstrip()
            else:
                logger.warning('Invalid setup function ignored')
                func = None
                args = ()
                kwargs = {}

        coro = asyncoro.Coro(_setup, self, peer, timeout, func, name, args,
                             kwargs)
        yield coro.finish()
コード例 #2
0
ファイル: discoro.py プロジェクト: yonglehou/asyncoro
    def run(self, peer, *args, **kwargs):
        """Once 'setup', the computation can be run as many times as
        necessary. Each 'run' creates a coroutine at peer and returns
        reference to that remote coroutine if successful; otherwise,
        gives -1.

        Must be used with yield as 'rcoro = yield computation.run(peer, ...)'
        """
        def _run(self, peer, coro=None):
            peer.send({
                'cmd': 'run',
                'client': coro,
                'computation': self._name,
                'args': args,
                'kwargs': kwargs
            })
            yield coro.receive(timeout=5)

        coro = asyncoro.Coro(_run, self, peer)
        yield coro.finish()
コード例 #3
0
def send_proc(coro=None):
    # if server is in a remote network, use 'peer' as (optionally
    # enabling streaming for efficiency):
    # yield asyncoro.AsynCoro.instance().peer('server node/ip')
    server = yield asyncoro.Coro.locate('server')
    yield server.send(('join', coro))
    client_id = yield coro.receive()

    channel = yield asyncoro.Channel.locate('channel')
    recv_coro = asyncoro.Coro(recv_proc, client_id)
    yield channel.subscribe(recv_coro)
    # since readline is synchronous (blocking) call, use async thread
    async_threads = asyncoro.AsyncThreadPool(1)
    while True:
        msg = yield async_threads.async_task(sys.stdin.readline)
        msg = msg.strip()
        if msg == 'quit':
            break
        channel.send((msg, client_id))
    server.send(('quit', client_id))
    yield channel.unsubscribe(recv_coro)
コード例 #4
0
            continue
        print('    %s %s' % (who, msg))


def send_proc(coro=None):
    # if server is in a remote network, use 'peer' as (optionally
    # enabling streaming for efficiency):
    # yield asyncoro.AsynCoro.instance().peer('server node/ip')
    server = yield asyncoro.Coro.locate('server')
    yield server.send(('join', coro))
    client_id = yield coro.receive()

    channel = yield asyncoro.Channel.locate('channel')
    recv_coro = asyncoro.Coro(recv_proc, client_id)
    yield channel.subscribe(recv_coro)
    # since readline is synchronous (blocking) call, use async thread
    async_threads = asyncoro.AsyncThreadPool(1)
    while True:
        msg = yield async_threads.async_task(sys.stdin.readline)
        msg = msg.strip()
        if msg == 'quit':
            break
        channel.send((msg, client_id))
    server.send(('quit', client_id))
    yield channel.unsubscribe(recv_coro)


if __name__ == '__main__':
    # asyncoro.logger.setLevel(logging.DEBUG)
    asyncoro.Coro(send_proc)
コード例 #5
0
ファイル: discoro.py プロジェクト: yonglehou/asyncoro
def _discoro_process(_discoro_config, _discoro_queue):
    _discoro_scheduler = asyncoro.AsynCoro(**_discoro_config)
    _discoro_coro = asyncoro.Coro(discoro_server)
    _discoro_queue.put(asyncoro.serialize(_discoro_coro))
    _discoro_scheduler.finish()
コード例 #6
0
ファイル: discoro.py プロジェクト: yonglehou/asyncoro
def discoro_server(_discoro_name='discoro_server'):
    """Server that receives computations and runs coroutines for it.
    """

    _discoro_coro = asyncoro.AsynCoro.cur_coro()
    _discoro_coro.register(_discoro_name)
    asyncoro.logger.debug('discoro %s started' % _discoro_coro)
    # os.chdir(asyncoro.AsynCoro.instance().dest_path)
    _discoro_computations = {}
    _discoro_globals = _discoro_locals = _discoro_cmd = _discoro_client = None
    _discoro_msg = _discoro_computation = _discoro_job_coro = None
    _discoro_job_coros = set()
    _discoro_globals = list(globals().keys())
    _discoro_locals = list(locals().keys())
    _discoro_globals.extend(_discoro_locals)

    while True:
        _discoro_msg = yield _discoro_coro.receive()
        try:
            _discoro_cmd = _discoro_msg['cmd']
        except:
            if isinstance(_discoro_msg, asyncoro.MonitorException):
                asyncoro.logger.debug('job %s done' % _discoro_msg.args[0])
                _discoro_job_coros.discard(_discoro_msg.args[0])
                continue
            else:
                asyncoro.logger.debug(traceback.format_exc())
                _discoro_cmd = None
        if _discoro_cmd == 'run':
            _discoro_client = _discoro_msg.get('client', None)
            if not isinstance(_discoro_client, asyncoro.Coro):
                continue
            try:
                _discoro_computation = _discoro_computations[
                    _discoro_msg['computation']]
                job_coro = asyncoro.Coro(globals()[_discoro_computation._name],
                                         *_discoro_msg['args'],
                                         **_discoro_msg['kwargs'])
            except:
                asyncoro.logger.debug('invalid computation to run')
                asyncoro.logger.debug(traceback.format_exc())
                job_coro = -1
            else:
                asyncoro.logger.debug('job %s created' % job_coro)
                _discoro_job_coros.add(job_coro)
                if (yield _discoro_coro.monitor(job_coro)) != 0:
                    _discoro_job_coros.discard(job_coro)
            _discoro_client.send(job_coro)
            del job_coro
        elif _discoro_cmd == 'ping':
            # TODO: maintain heartbeat times and remove computations from dead clients?
            pass
        elif _discoro_cmd == 'setup':
            _discoro_client = _discoro_msg.get('client', None)
            if not isinstance(_discoro_client, asyncoro.Coro):
                continue
            try:
                _discoro_computation = _discoro_msg['computation']
                exec(compile(_discoro_computation._code, '<string>', 'exec'))
                globals().update(locals())
                if _discoro_msg['func']:
                    try:
                        exec(compile(_discoro_msg['func'], '<string>', 'exec'))
                        locals()[_discoro_msg['name']](
                            *_discoro_msg['args'], **_discoro_msg['kwargs'])
                    except:
                        logger.warning('setup function failed: %s' %
                                       traceback.format_exc())
            except:
                asyncoro.logger.warning('invalid computation')
                asyncoro.logger.debug(traceback.format_exc())
                _discoro_client.send(-1)
                continue
            asyncoro.logger.debug('computation "%s" from "%s"' % \
                                  (_discoro_computation._name, _discoro_msg['client']))
            setattr(_discoro_computation, '_client', _discoro_client)
            setattr(_discoro_computation, '_globals',
                    [_discoro_var for _discoro_var in globals().keys() \
                     if _discoro_var not in _discoro_globals])
            setattr(_discoro_computation, '_locals',
                    [_discoro_var for _discoro_var in locals().keys() \
                     if _discoro_var not in _discoro_locals])
            _discoro_globals.extend(_discoro_computation._globals)
            _discoro_locals.extend(_discoro_computation._locals)
            _discoro_computations[
                _discoro_computation._name] = _discoro_computation
            _discoro_client.send(0)
        elif _discoro_cmd == 'close':
            try:
                _discoro_computation = _discoro_computations.pop(
                    _discoro_msg['computation'], None)
                asyncoro.logger.debug('deleting computation "%s"' %
                                      _discoro_computation._name)
                if _discoro_msg['func']:
                    try:
                        exec(compile(_discoro_msg['func'], '<string>', 'exec'))
                        locals()[_discoro_msg['name']](
                            *_discoro_msg['args'], **_discoro_msg['kwargs'])
                    except:
                        logger.warning('cleanup function failed: %s' %
                                       traceback.format_exc())
                for _discoro_var in _discoro_computation._globals:
                    globals().pop(_discoro_var, None)
                for _discoro_var in _discoro_computation._locals:
                    locals().pop(_discoro_var, None)
            except:
                asyncoro.logger.warning('invalid computation "%s" to delete' % \
                                        _discoro_msg.get('computation', None))
        elif _discoro_cmd == 'quit':
            break
        else:
            asyncoro.logger.warning('invalid command "%s" ignored' %
                                    _discoro_cmd)
            _discoro_client = _discoro_msg.get('client', None)
            if not isinstance(_discoro_client, asyncoro.Coro):
                continue
            _discoro_client.send(-1)

    # wait until all computations are done; process only 'ping' and 'close'
    while _discoro_job_coros:
        _discoro_msg = yield _discoro_coro.receive()
        try:
            _discoro_cmd = _discoro_msg['cmd']
        except:
            if isinstance(_discoro_msg, asyncoro.MonitorException):
                asyncoro.logger.debug('job %s done' % _discoro_msg.args[0])
                _discoro_job_coros.discard(_discoro_msg.args[0])
                continue
            else:
                asyncoro.logger.debug(traceback.format_exc())
            _discoro_cmd = None
        if _discoro_cmd == 'ping':
            # TODO: maintain heartbeat times and remove computations from dead clients?
            pass
        elif _discoro_cmd == 'close':
            try:
                _discoro_computation = _discoro_computations.pop(
                    _discoro_msg['computation'], None)
                asyncoro.logger.debug('deleting computation "%s"' %
                                      _discoro_computation._name)
                for _discoro_var in _discoro_computation._globals:
                    globals().pop(_discoro_var, None)
                for _discoro_var in _discoro_computation._locals:
                    locals().pop(_discoro_var, None)
                if not _discoro_computations:
                    break
            except:
                asyncoro.logger.warning('invalid computation to delete')
        else:
            asyncoro.logger.warning('invalid command "%s" ignored' %
                                    _discoro_cmd)
            _discoro_client = _discoro_msg.get('client', None)
            if not isinstance(_discoro_client, asyncoro.Coro):
                continue
            _discoro_client.send(-1)

    for _discoro_computation in _discoro_computations.values():
        asyncoro.logger.debug('deleting computation "%s"' %
                              _discoro_computation._name)
        for _discoro_var in _discoro_computation._globals:
            globals().pop(_discoro_var, None)
        for _discoro_var in _discoro_computation._locals:
            locals().pop(_discoro_var, None)

    for _discoro_var in _discoro_locals:
        if _discoro_var == '_discoro_locals':
            continue
        globals().pop(_discoro_var, None)
    globals().pop('_discoro_locals', None)
コード例 #7
0
ファイル: discoro.py プロジェクト: yonglehou/asyncoro
    _discoro_name = _discoro_config['name']
    _discoro_queue = multiprocessing.Queue()
    _discoro_processes = []
    for _discoro_proc_id in range(2, _discoro_cpus + 1):
        _discoro_config['name'] = _discoro_name + '-%s' % _discoro_proc_id
        _discoro_processes.append(
            multiprocessing.Process(target=_discoro_process,
                                    args=(_discoro_config, _discoro_queue)))
        _discoro_processes[-1].start()
        time.sleep(0.05)

    _discoro_proc_id = 1
    _discoro_config['name'] = _discoro_name + '-%s' % _discoro_proc_id
    _discoro_scheduler = asyncoro.AsynCoro(**_discoro_config)
    _discoro_coro = asyncoro.Coro(discoro_server)

    while True:
        try:
            if sys.stdin.readline().strip().lower() in ('quit', 'exit'):
                break
        except KeyboardInterrupt:
            break

    asyncoro.logger.debug('terminating servers')
    while not _discoro_queue.empty():
        asyncoro.unserialize(_discoro_queue.get()).send({'cmd': 'quit'})
    _discoro_coro.send({'cmd': 'quit'})

    for _i, _discoro_proc in enumerate(_discoro_processes, start=1):
        if _discoro_proc.is_alive():
コード例 #8
0
    coro.register('server')
    client_id = 1
    while True:
        cmd, who = yield coro.receive()
        # join/quit messages can be sent by clients themselves, but
        # for illustration server sends them instead
        if cmd == 'join':
            channel.send(('joined', client_id))
            who.send(client_id)
            client_id += 1
        elif cmd == 'quit':
            channel.send(('bye', who))
        elif cmd == 'terminate':
            break
    channel.unregister()
    coro.unregister()


if __name__ == '__main__':
    # asyncoro.logger.setLevel(logging.DEBUG)
    server = asyncoro.Coro(server_proc)
    while True:
        try:
            cmd = sys.stdin.readline().strip()
            if cmd == 'quit':
                break
        except KeyboardInterrupt:
            break
    server.send(('terminate', None))
    server.value()  # wait for server to finish