コード例 #1
0
    def _connectToEndpoint(self, host, port, timeout, blocking=False):
        if self.isConnected():
            raise IllegalStateError(
                'service "{0}" is already connected'.format(self.name))

        addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
        if not addressInfoList:
            raise ConnectionResolveError((host, port))

        pipe_timeout = float(timeout) / len(
            addressInfoList) if timeout is not None else None

        log.debug('Connecting to the service "{0}", candidates: {1}'.format(
            self.name, addressInfoList))
        start = time()
        errors = []
        for family, socktype, proto, canonname, address in addressInfoList:
            log.debug(' - connecting to "{0} {1}"'.format(proto, address))
            sock = socket.socket(family=family, type=socktype, proto=proto)
            try:
                self._pipe = Pipe(sock)
                yield self._pipe.connect(address,
                                         timeout=pipe_timeout,
                                         blocking=blocking)
                log.debug(' - success')
            except ConnectionError as err:
                errors.append(err)
                log.debug(' - failed - {0}'.format(err))
            except Exception as err:
                log.warn(
                    'Unexpected error caught while connecting to the "{0}" - {1}'
                    .format(address, err))
            else:
                self._ioLoop = self._pipe._ioLoop
                self._writableStream = WritableStream(self._ioLoop, self._pipe)
                self._readableStream = ReadableStream(self._ioLoop, self._pipe)
                self._ioLoop.bind_on_fd(self._pipe.fileno())

                def decode_and_dispatch(on_event):
                    def dispatch(unpacker):
                        for chunk in unpacker:
                            on_event(chunk)

                    return dispatch

                self._readableStream.bind(decode_and_dispatch(
                    self._on_message))
                return

        if timeout is not None and time() - start > timeout:
            raise ConnectionTimeoutError((host, port), timeout)

        prefix = 'service resolving failed. Reason:'
        reason = '{0} [{1}]'.format(prefix,
                                    ', '.join(str(err) for err in errors))
        raise ConnectionError((host, port), reason)
コード例 #2
0
    def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs):
        """
        It:
        * goes to Locator and get service api (put into self._service_api)
        * initializes event loop and all asio elements (write/read streams)
        * initializes session counter
        * registers callback on epoll READ event and\
        binds callback to decoder (_on_message)
        """
        if '--locator' in init_args:
            try:
                port = int(init_args[init_args.index('--locator') + 1])
            except ValueError as err:
                port = 10053
            except IndexError as err:
                port = 10053
        
        self._try_reconnect = kwargs.get("reconnect_once", True)
        self._counter = 1
        self.loop = ev.Loop()

        self._locator_host = endpoint
        self._locator_port = port
        self.servicename = name

        self._init_endpoint() # initialize pipe

        self.decoder = Decoder()
        self.decoder.bind(self._on_message)

        self.w_stream = WritableStream(self.loop, self.pipe)
        self.r_stream = ReadableStream(self.loop, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno())
コード例 #3
0
    def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
        self._logger = core_log
        self._init_endpoint(init_args or sys.argv)

        self.sessions = dict()
        self.sandbox = Sandbox()

        self.loop = ev.Loop()

        self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
        self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
        self.heartbeat_timer.start()

        if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
            if len(self.endpoint) == 2:
                socket_type = socket.AF_INET
            elif len(self.endpoint) == 4:
                socket_type = socket.AF_INET6
            else:
                raise ValueError('invalid endpoint')
        elif isinstance(self.endpoint, types.StringType):
            socket_type = socket.AF_UNIX
        else:
            raise ValueError('invalid endpoint')
        sock = socket.socket(socket_type)
        self.pipe = Pipe(sock)
        self.pipe.connect(self.endpoint, blocking=True)
        self.loop.bind_on_fd(self.pipe.fileno())

        self.decoder = Decoder()
        self.decoder.bind(self.on_message)

        self.w_stream = WritableStream(self.loop, self.pipe)
        self.r_stream = ReadableStream(self.loop, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.loop.register_read_event(self.r_stream._on_event,
                                      self.pipe.fileno())
        self._logger.debug("Worker with %s send handshake" % self.id)
        # Send both messages - to run timers properly. This messages will be sent
        # only after all initialization, so they have same purpose.
        self._send_handshake()
        self._send_heartbeat()
コード例 #4
0
    def _connectToEndpoint(self, host, port, timeout, blocking=False):
        if self.isConnected():
            raise IllegalStateError('service "{0}" is already connected'.format(self.name))

        addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
        if not addressInfoList:
            raise ConnectionResolveError((host, port))

        pipe_timeout = float(timeout) / len(addressInfoList) if timeout is not None else None

        log.debug('Connecting to the service "{0}", candidates: {1}'.format(self.name, addressInfoList))
        start = time()
        errors = []
        for family, socktype, proto, canonname, address in addressInfoList:
            log.debug(' - connecting to "{0} {1}"'.format(proto, address))
            sock = socket.socket(family=family, type=socktype, proto=proto)
            try:
                self._pipe = Pipe(sock)
                yield self._pipe.connect(address, timeout=pipe_timeout, blocking=blocking)
                log.debug(' - success')
            except ConnectionError as err:
                errors.append(err)
                log.debug(' - failed - {0}'.format(err))
            except Exception as err:
                log.warn('Unexpected error caught while connecting to the "{0}" - {1}'.format(address, err))
            else:
                self._ioLoop = self._pipe._ioLoop
                self._writableStream = WritableStream(self._ioLoop, self._pipe)
                self._readableStream = ReadableStream(self._ioLoop, self._pipe)
                self._ioLoop.bind_on_fd(self._pipe.fileno())

                def decode_and_dispatch(on_event):
                    def dispatch(unpacker):
                        for chunk in unpacker:
                            on_event(chunk)
                    return dispatch
                self._readableStream.bind(decode_and_dispatch(self._on_message))
                return

        if timeout is not None and time() - start > timeout:
            raise ConnectionTimeoutError((host, port), timeout)

        prefix = 'service resolving failed. Reason:'
        reason = '{0} [{1}]'.format(prefix, ', '.join(str(err) for err in errors))
        raise ConnectionError((host, port), reason)
コード例 #5
0
    def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv):
        def closure(number):
            def wrapper(*args):
                def register_callback(callback, errorback=None):
                    self.w_stream.write([number, self._counter, args])
                    self._subscribers[self._counter] = (callback, errorback)
                    # FIX: Move counter increment for supporting stream-loke services
                    self._counter += 1
                return register_callback
            return wrapper

        service_endpoint, _, service_api = self._get_api(name, endpoint, port)

        self._service_api = service_api
        self.servicename = name

        for number, methodname in service_api.iteritems():
            setattr(self, methodname, closure(number))

        self.service = ev.Service()

        self._counter = 1
        self._subscribers = dict()

        self.pipe = ServicePipe(service_endpoint)

        self.decoder = Decoder()
        self.decoder.bind(self._on_message)

        self.service.bind_on_fd(self.pipe.fileno())

        self.w_stream = WritableStream(self.service, self.pipe)
        self.r_stream = ReadableStream(self.service, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno())
        try:
            self.app_name = init_args[init_args.index("--app") + 1]
        except ValueError:
            self.app_name = "standalone"
コード例 #6
0
    def _connectToEndpoint(self, host, port, timeout, blocking=False):
        if self.isConnected():
            raise IllegalStateError('service "{0}" is already connected'.format(self.name))

        addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
        if not addressInfoList:
            raise ConnectionResolveError((host, port))

        start = time()
        errors = []
        for family, socktype, proto, canonname, address in addressInfoList:
            sock = socket.socket(family=family, type=socktype, proto=proto)
            try:
                self._pipe = Pipe(sock)
                remainingTimeout = timeout - (time() - start) if timeout is not None else None
                yield self._pipe.connect(address, timeout=remainingTimeout, blocking=blocking)
            except ConnectionError as err:
                errors.append(err)
            else:
                self._ioLoop = self._pipe._ioLoop
                #todo: Is we REALLY need to reconnect streams instead of creating new ones?
                self._writableStream = WritableStream(self._ioLoop, self._pipe)
                self._readableStream = ReadableStream(self._ioLoop, self._pipe)
                self._ioLoop.bind_on_fd(self._pipe.fileno())

                def decode_and_dispatch(on_event):
                    def dispatch(unpacker):
                        for chunk in unpacker:
                            on_event(chunk)
                    return dispatch
                self._readableStream.bind(decode_and_dispatch(self._on_message))
                return

        if timeout is not None and time() - start > timeout:
            raise ConnectionTimeoutError((host, port), timeout)

        prefix = 'service resolving failed. Reason:'
        reason = '{0} [{1}]'.format(prefix, ', '.join(str(err) for err in errors))
        raise ConnectionError((host, port), reason)
コード例 #7
0
class AbstractService(object):
    """Represents abstract cocaine service.

    It provides basic service operations like getting its actual network address, determining if the service is
    connecting or connected.

    There is no other useful public methods, so the main aim of this class - is to provide superclass for inheriting
    for actual services or service-like objects (i.e. Locator).

    :ivar name: service name.
    """
    def __init__(self, name):
        self.name = name

        self._pipe = None
        self._ioLoop = None
        self._writableStream = None
        self._readableStream = None

        self._subscribers = {}
        self._session = 0

        self.version = 0
        self.api = {}

    @property
    def address(self):
        """Return actual network address (`sockaddr`) of the current service if it is connected.

        Returned `sockaddr` is a tuple describing a socket address, whose format depends on the returned
        family `(address, port)` 2-tuple for AF_INET, or `(address, port, flow info, scope id)` 4-tuple for AF_INET6),
        and is meant to be passed to the socket.connect() method.

        It the service is not connected this method returns tuple `('NOT_CONNECTED', 0)`.
        """
        return self._pipe.address if self.isConnected() else ('NOT_CONNECTED', 0)

    def isConnecting(self):
        """Return true if the service is in connecting state."""
        return self._pipe is not None and self._pipe.isConnecting()

    def isConnected(self):
        """Return true if the service is in connected state."""
        return self._pipe is not None and self._pipe.isConnected()

    def disconnect(self):
        """Disconnect service from its endpoint and destroys all communications between them.

        .. note:: This method does nothing if the service is not connected.
        """
        if not self._pipe:
            return
        self._pipe.close()
        self._pipe = None

    @strategy.coroutine
    def _connectToEndpoint(self, host, port, timeout, blocking=False):
        if self.isConnected():
            raise IllegalStateError('service "{0}" is already connected'.format(self.name))

        addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
        if not addressInfoList:
            raise ConnectionResolveError((host, port))

        pipe_timeout = float(timeout) / len(addressInfoList) if timeout is not None else None

        log.debug('Connecting to the service "{0}", candidates: {1}'.format(self.name, addressInfoList))
        start = time()
        errors = []
        for family, socktype, proto, canonname, address in addressInfoList:
            log.debug(' - connecting to "{0} {1}"'.format(proto, address))
            sock = socket.socket(family=family, type=socktype, proto=proto)
            try:
                self._pipe = Pipe(sock)
                yield self._pipe.connect(address, timeout=pipe_timeout, blocking=blocking)
                log.debug(' - success')
            except ConnectionError as err:
                errors.append(err)
                log.debug(' - failed - {0}'.format(err))
            except Exception as err:
                log.warn('Unexpected error caught while connecting to the "{0}" - {1}'.format(address, err))
            else:
                self._ioLoop = self._pipe._ioLoop
                self._writableStream = WritableStream(self._ioLoop, self._pipe)
                self._readableStream = ReadableStream(self._ioLoop, self._pipe)
                self._ioLoop.bind_on_fd(self._pipe.fileno())

                def decode_and_dispatch(on_event):
                    def dispatch(unpacker):
                        for chunk in unpacker:
                            on_event(chunk)
                    return dispatch
                self._readableStream.bind(decode_and_dispatch(self._on_message))
                return

        if timeout is not None and time() - start > timeout:
            raise ConnectionTimeoutError((host, port), timeout)

        prefix = 'service resolving failed. Reason:'
        reason = '{0} [{1}]'.format(prefix, ', '.join(str(err) for err in errors))
        raise ConnectionError((host, port), reason)

    def _on_message(self, args):
        msg = Message.initialize(args)
        if msg is None:
            return

        try:
            if msg.id == message.RPC_CHUNK:
                self._subscribers[msg.session].trigger(msgpack.unpackb(msg.data))
            elif msg.id == message.RPC_CHOKE:
                future = self._subscribers.pop(msg.session, None)
                assert future is not None, 'one of subscribers has suddenly disappeared'
                if future is not None:
                    future.close()
            elif msg.id == message.RPC_ERROR:
                self._subscribers[msg.session].error(ServiceError(self.name, msg.message, msg.code))
        except Exception as err:
            log.warning('"_on_message" method has caught an error - %s', err)
            raise err

    def _invoke(self, methodId):
        def wrapper(*args, **kwargs):
            future = Deferred()
            timeout = kwargs.get('timeout', None)
            if timeout is not None:
                timeoutId = self._ioLoop.add_timeout(time() + timeout, lambda: future.error(TimeoutError(timeout)))

                def timeoutRemover(func):
                    def wrapper(*args, **kwargs):
                        self._ioLoop.remove_timeout(timeoutId)
                        return func(*args, **kwargs)
                    return wrapper
                future.close = timeoutRemover(future.close)
            self._session += 1
            self._writableStream.write([methodId, self._session, args])
            self._subscribers[self._session] = future
            return Chain([lambda: future], ioLoop=self._ioLoop)
        return wrapper

    def perform_sync(self, method, *args, **kwargs):
        """Performs synchronous method invocation via direct socket usage without the participation of the event loop.

        Returns generator of chunks.

        :param method: method name.
        :param args: method arguments.
        :param kwargs: method keyword arguments. You can specify `timeout` keyword to set socket timeout.

        .. note:: Left for backward compatibility, tests and other stuff. Indiscriminate using of this method can lead
                  to the summoning of Satan.
        .. warning:: Do not mix synchronous and asynchronous usage of service!
        """
        if not self.isConnected():
            raise IllegalStateError('service "{0}" is not connected'.format(self.name))

        if method not in self.api:
            raise ValueError('service "{0}" has no method named "{1}"'.format(self.name, method))

        timeout = kwargs.get('timeout', None)
        if timeout is not None and timeout <= 0:
            raise ValueError('timeout must be positive number')

        with scope.socket.timeout(self._pipe.sock, timeout) as sock:
            self._session += 1
            sock.send(msgpack.dumps([self.api[method], self._session, args]))
            unpacker = msgpack.Unpacker()
            error = None
            while True:
                data = sock.recv(4096)
                unpacker.feed(data)
                for chunk in unpacker:
                    msg = Message.initialize(chunk)
                    if msg is None:
                        continue
                    if msg.id == message.RPC_CHUNK:
                        yield msgpack.loads(msg.data)
                    elif msg.id == message.RPC_CHOKE:
                        raise error or StopIteration
                    elif msg.id == message.RPC_ERROR:
                        error = ServiceError(self.name, msg.message, msg.code)
コード例 #8
0
class BaseService(object):
    """ Implements basic functional for services:
    * all asio stuff
    * perform_sync method for synchronous operations
    You should reimplement _on_message function - this is callback for decoder,
    so this function is called with every incoming decoded message
    """

    def __init__(self, name, endpoint="127.0.0.1", port=10053, init_args=sys.argv, **kwargs):
        """
        It:
        * goes to Locator and get service api (put into self._service_api)
        * initializes event loop and all asio elements (write/read streams)
        * initializes session counter
        * registers callback on epoll READ event and\
        binds callback to decoder (_on_message)
        """
        if '--locator' in init_args:
            try:
                port = int(init_args[init_args.index('--locator') + 1])
            except ValueError as err:
                port = 10053
            except IndexError as err:
                port = 10053
        
        self._try_reconnect = kwargs.get("reconnect_once", True)
        self._counter = 1
        self.loop = ev.Loop()

        self._locator_host = endpoint
        self._locator_port = port
        self.servicename = name

        self._init_endpoint() # initialize pipe

        self.decoder = Decoder()
        self.decoder.bind(self._on_message)

        self.w_stream = WritableStream(self.loop, self.pipe)
        self.r_stream = ReadableStream(self.loop, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno())

    def _init_endpoint(self):
        locator = Locator()
        self.service_endpoint, _, service_api = locator.resolve(self.servicename, self._locator_host, self._locator_port)
        self._service_api = service_api
        # msgpack convert in list or tuple depend on version - make it tuple
        self.pipe = Pipe(tuple(self.service_endpoint), self.reconnect if self._try_reconnect else None)
        self.loop.bind_on_fd(self.pipe.fileno())
        
    def reconnect(self):
        self.loop.stop_listening(self.pipe.fileno())
        #try:
        #    self.pipe.sock.close()
        #except Exception as err:
        #    print(str(err))
        try:
            self._init_endpoint()
        except LocatorResolveError as err:
            pass
        else:
            self.w_stream.reconnect(self.pipe)
            self.r_stream.reconnect(self.pipe)

    def perform_sync(self, method, *args, **kwargs):
        """ Do not use the service synchronously after treatment to him asynchronously!
        Use for these purposes the other instance of the service!
        """

        timeout = kwargs.get("timeout", 5)

        # Get number of current method
        try:
            number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next()
        except StopIteration as err:
            raise ServiceError(self.servicename, "method %s is not available" % method, -100)

        try:
            # DO IT SYNC
            self.pipe.settimeout(timeout)
            self.pipe.writeall(packb([number, self._counter, args]))
            self._counter += 1
            u = Unpacker()
            msg = None

            # If we receive rpc::error, put ServiceError here, 
            # and raise this error instead of StopIteration on rpc::choke,
            # because after rpc::error we always receive choke.
            _error = None

            while True:
                response = self.pipe.recv(4096)
                u.feed(response)
                for _data in u:
                    msg = Message.initialize(_data)
                    if msg is None:
                        continue
                    if msg.id == message.RPC_CHUNK:
                        yield unpackb(msg.data)
                    elif msg.id == message.RPC_CHOKE:
                        raise _error or StopIteration
                    elif msg.id == message.RPC_ERROR:
                        _error = ServiceError(self.servicename, msg.message, msg.code)
        finally:
            self.pipe.settimeout(0)  # return to non-blocking mode

    def _on_message(self, args):
        raise NotImplementedError()

    @property
    def connected(self):
        return self.pipe.connected
コード例 #9
0
class Worker(object):

    def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
        self._logger = core_log
        self._init_endpoint(init_args or sys.argv)

        self.sessions = dict()
        self.sandbox = Sandbox()

        self.loop = ev.Loop()

        self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
        self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
        self.heartbeat_timer.start()

        if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
            if len(self.endpoint) == 2:
                socket_type = socket.AF_INET
            elif len(self.endpoint) == 4:
                socket_type = socket.AF_INET6
            else:
                raise ValueError('invalid endpoint')
        elif isinstance(self.endpoint, types.StringType):
            socket_type = socket.AF_UNIX
        else:
            raise ValueError('invalid endpoint')
        sock = socket.socket(socket_type)
        self.pipe = Pipe(sock)
        self.pipe.connect(self.endpoint, blocking=True)
        self.loop.bind_on_fd(self.pipe.fileno())

        self.decoder = Decoder()
        self.decoder.bind(self.on_message)

        self.w_stream = WritableStream(self.loop, self.pipe)
        self.r_stream = ReadableStream(self.loop, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.loop.register_read_event(self.r_stream._on_event,
                                      self.pipe.fileno())
        self._logger.debug("Worker with %s send handshake" % self.id)
        # Send both messages - to run timers properly. This messages will be sent
        # only after all initialization, so they have same purpose.
        self._send_handshake()

    def _init_endpoint(self, init_args):
        try:
            self.id = init_args[init_args.index("--uuid") + 1]
            # app_name = init_args[init_args.index("--app") + 1]
            self.endpoint = init_args[init_args.index("--endpoint") + 1]
        except Exception as err:
            self._logger.error("Wrong cmdline arguments: %s " % err)
            raise RuntimeError("Wrong cmdline arguments")

    def run(self, binds=None):
        if not binds:
            binds = {}
        for event, name in binds.iteritems():
            self.on(event, name)
        self._send_heartbeat()
        self.loop.run()

    def terminate(self, reason, msg):
        self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
        self.loop.stop()
        exit(1)

    # Event machine
    def on(self, event, callback):
        self.sandbox.on(event, callback)

    # Events
    def on_heartbeat(self):
        self._send_heartbeat()

    def on_message(self, args):
        msg = Message.initialize(args)
        if msg is None:
            return

        elif msg.id == message.RPC_INVOKE:
            request = Request()
            stream = Stream(msg.session, self, msg.event)
            try:
                self.sandbox.invoke(msg.event, request, stream)
                self.sessions[msg.session] = request
            except (ImportError, SyntaxError) as err:
                stream.error(2, "unrecoverable error: %s " % str(err))
                self.terminate(1, "Bad code")
            except Exception as err:
                self._logger.error("On invoke error: %s" % err)
                traceback.print_stack()
                stream.error(1, "Invocation error")

        elif msg.id == message.RPC_CHUNK:
            self._logger.debug("Receive chunk: %d" % msg.session)
            try:
                _session = self.sessions[msg.session]
                _session.push(msg.data)
            except Exception as err:
                self._logger.error("On push error: %s" % str(err))
                self.terminate(1, "Push error: %s" % str(err))
                return

        elif msg.id == message.RPC_CHOKE:
            self._logger.debug("Receive choke: %d" % msg.session)
            _session = self.sessions.get(msg.session, None)
            if _session is not None:
                _session.close()
                self.sessions.pop(msg.session)

        elif msg.id == message.RPC_HEARTBEAT:
            self._logger.debug("Receive heartbeat. Stop disown timer")
            self.disown_timer.stop()

        elif msg.id == message.RPC_TERMINATE:
            self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
            self.terminate(msg.reason, msg.message)

        elif msg.id == message.RPC_ERROR:
            _session = self.sessions.get(msg.session, None)
            if _session is not None:
                _session.error(RequestError(msg.message))

    def on_disown(self):
        try:
            self._logger.error("Disowned")
        finally:
            self.loop.stop()

    # Private:
    def _send_handshake(self):
        self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())

    def _send_heartbeat(self):
        self.disown_timer.start()
        self._logger.debug("Send heartbeat. Start disown timer")
        self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())

    def send_choke(self, session):
        self.w_stream.write(Message(message.RPC_CHOKE, session).pack())

    def send_chunk(self, session, data):
        self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())

    def send_error(self, session, code, msg):
        self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
コード例 #10
0
class AbstractService(object):
    """Represents abstract cocaine service.

    It provides basic service operations like getting its actual network address, determining if the service is
    connecting or connected.

    There is no other useful public methods, so the main aim of this class - is to provide superclass for inheriting
    for actual services or service-like objects (i.e. Locator).

    :ivar name: service name.
    """
    def __init__(self, name):
        self.name = name

        self._pipe = None
        self._ioLoop = None
        self._writableStream = None
        self._readableStream = None

        self._subscribers = {}
        self._session = 0

        self.version = 0
        self.api = {}

    @property
    def address(self):
        """Return actual network address (`sockaddr`) of the current service if it is connected.

        Returned `sockaddr` is a tuple describing a socket address, whose format depends on the returned
        family `(address, port)` 2-tuple for AF_INET, or `(address, port, flow info, scope id)` 4-tuple for AF_INET6),
        and is meant to be passed to the socket.connect() method.

        It the service is not connected this method returns tuple `('NOT_CONNECTED', 0)`.
        """
        return self._pipe.address if self.isConnected() else ('NOT_CONNECTED',
                                                              0)

    def isConnecting(self):
        """Return true if the service is in connecting state."""
        return self._pipe is not None and self._pipe.isConnecting()

    def isConnected(self):
        """Return true if the service is in connected state."""
        return self._pipe is not None and self._pipe.isConnected()

    def disconnect(self):
        """Disconnect service from its endpoint and destroys all communications between them.

        .. note:: This method does nothing if the service is not connected.
        """
        if not self._pipe:
            return
        self._pipe.close()
        self._pipe = None

    @strategy.coroutine
    def _connectToEndpoint(self, host, port, timeout, blocking=False):
        if self.isConnected():
            raise IllegalStateError(
                'service "{0}" is already connected'.format(self.name))

        addressInfoList = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
        if not addressInfoList:
            raise ConnectionResolveError((host, port))

        pipe_timeout = float(timeout) / len(
            addressInfoList) if timeout is not None else None

        log.debug('Connecting to the service "{0}", candidates: {1}'.format(
            self.name, addressInfoList))
        start = time()
        errors = []
        for family, socktype, proto, canonname, address in addressInfoList:
            log.debug(' - connecting to "{0} {1}"'.format(proto, address))
            sock = socket.socket(family=family, type=socktype, proto=proto)
            try:
                self._pipe = Pipe(sock)
                yield self._pipe.connect(address,
                                         timeout=pipe_timeout,
                                         blocking=blocking)
                log.debug(' - success')
            except ConnectionError as err:
                errors.append(err)
                log.debug(' - failed - {0}'.format(err))
            except Exception as err:
                log.warn(
                    'Unexpected error caught while connecting to the "{0}" - {1}'
                    .format(address, err))
            else:
                self._ioLoop = self._pipe._ioLoop
                self._writableStream = WritableStream(self._ioLoop, self._pipe)
                self._readableStream = ReadableStream(self._ioLoop, self._pipe)
                self._ioLoop.bind_on_fd(self._pipe.fileno())

                def decode_and_dispatch(on_event):
                    def dispatch(unpacker):
                        for chunk in unpacker:
                            on_event(chunk)

                    return dispatch

                self._readableStream.bind(decode_and_dispatch(
                    self._on_message))
                return

        if timeout is not None and time() - start > timeout:
            raise ConnectionTimeoutError((host, port), timeout)

        prefix = 'service resolving failed. Reason:'
        reason = '{0} [{1}]'.format(prefix,
                                    ', '.join(str(err) for err in errors))
        raise ConnectionError((host, port), reason)

    def _on_message(self, args):
        msg = Message.initialize(args)
        if msg is None:
            return

        try:
            if msg.id == message.RPC_CHUNK:
                self._subscribers[msg.session].trigger(
                    msgpack.unpackb(msg.data))
            elif msg.id == message.RPC_CHOKE:
                future = self._subscribers.pop(msg.session, None)
                assert future is not None, 'one of subscribers has suddenly disappeared'
                if future is not None:
                    future.close()
            elif msg.id == message.RPC_ERROR:
                self._subscribers[msg.session].error(
                    ServiceError(self.name, msg.message, msg.code))
        except Exception as err:
            log.warning('"_on_message" method has caught an error - %s', err)
            raise err

    def _invoke(self, methodId):
        def wrapper(*args, **kwargs):
            future = Deferred()
            timeout = kwargs.get('timeout', None)
            if timeout is not None:
                timeoutId = self._ioLoop.add_timeout(
                    time() + timeout,
                    lambda: future.error(TimeoutError(timeout)))

                def timeoutRemover(func):
                    def wrapper(*args, **kwargs):
                        self._ioLoop.remove_timeout(timeoutId)
                        return func(*args, **kwargs)

                    return wrapper

                future.close = timeoutRemover(future.close)
            self._session += 1
            self._writableStream.write([methodId, self._session, args])
            self._subscribers[self._session] = future
            return Chain([lambda: future], ioLoop=self._ioLoop)

        return wrapper

    def perform_sync(self, method, *args, **kwargs):
        """Performs synchronous method invocation via direct socket usage without the participation of the event loop.

        Returns generator of chunks.

        :param method: method name.
        :param args: method arguments.
        :param kwargs: method keyword arguments. You can specify `timeout` keyword to set socket timeout.

        .. note:: Left for backward compatibility, tests and other stuff. Indiscriminate using of this method can lead
                  to the summoning of Satan.
        .. warning:: Do not mix synchronous and asynchronous usage of service!
        """
        if not self.isConnected():
            raise IllegalStateError('service "{0}" is not connected'.format(
                self.name))

        if method not in self.api:
            raise ValueError('service "{0}" has no method named "{1}"'.format(
                self.name, method))

        timeout = kwargs.get('timeout', None)
        if timeout is not None and timeout <= 0:
            raise ValueError('timeout must be positive number')

        with scope.socket.timeout(self._pipe.sock, timeout) as sock:
            self._session += 1
            sock.send(msgpack.dumps([self.api[method], self._session, args]))
            unpacker = msgpack.Unpacker()
            error = None
            while True:
                data = sock.recv(4096)
                unpacker.feed(data)
                for chunk in unpacker:
                    msg = Message.initialize(chunk)
                    if msg is None:
                        continue
                    if msg.id == message.RPC_CHUNK:
                        yield msgpack.loads(msg.data)
                    elif msg.id == message.RPC_CHOKE:
                        raise error or StopIteration
                    elif msg.id == message.RPC_ERROR:
                        error = ServiceError(self.name, msg.message, msg.code)
コード例 #11
0
class Service(object):

    def __init__(self, name, endpoint="localhost", port=10053, init_args=sys.argv):
        def closure(number):
            def wrapper(*args):
                def register_callback(callback, errorback=None):
                    self.w_stream.write([number, self._counter, args])
                    self._subscribers[self._counter] = (callback, errorback)
                    # FIX: Move counter increment for supporting stream-loke services
                    self._counter += 1
                return register_callback
            return wrapper

        service_endpoint, _, service_api = self._get_api(name, endpoint, port)

        self._service_api = service_api
        self.servicename = name

        for number, methodname in service_api.iteritems():
            setattr(self, methodname, closure(number))

        self.service = ev.Service()

        self._counter = 1
        self._subscribers = dict()

        self.pipe = ServicePipe(service_endpoint)

        self.decoder = Decoder()
        self.decoder.bind(self._on_message)

        self.service.bind_on_fd(self.pipe.fileno())

        self.w_stream = WritableStream(self.service, self.pipe)
        self.r_stream = ReadableStream(self.service, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno())
        try:
            self.app_name = init_args[init_args.index("--app") + 1]
        except ValueError:
            self.app_name = "standalone"

    def _get_api(self, name, endpoint, port):
        locator_pipe = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        locator_pipe.settimeout(4.0)
        locator_pipe.connect((endpoint, port))
        locator_pipe.send(packb([0, 1, [name]]))
        u = Unpacker()
        msg = None
        while msg is None:
            response = locator_pipe.recv(80960)
            u.feed(response)
            msg = Message.initialize(u.next())

        locator_pipe.close()
        if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
            return unpackb(msg.data)
        if msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
            raise Exception(msg.message)

    def perform_sync(self, method, *args, **kwargs):
        """ Do not use the service synchronously after treatment to him asynchronously!
        Use for these purposes the other instance of the service!
        """

        timeout = kwargs.get("timeout", 5)
        # Get number of current method
        try:
            number = (_num for _num, _name in self._service_api.iteritems() if _name == method).next()
        except StopIteration as err:
            raise ServiceError(self.servicename, "method %s is not available" % method, -100)

        try:
            self.pipe.settimeout(timeout) # DO IT SYNC
            self.pipe.writeall(packb([number, self._counter, args]))
            self._counter += 1
            u = Unpacker()
            msg = None

            # If we receive rpc::error, put ServiceError here, 
            # and raise this error instead of StopIteration on rpc::choke,
            # because after rpc::error we always receive choke.
            _error = None

            while True:
                response = self.pipe.recv(4096)
                u.feed(response)
                for _data in u:
                    msg = Message.initialize(_data)
                    if msg is None:
                        continue
                    if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
                        yield unpackb(msg.data)
                    elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"):
                        raise _error or StopIteration
                    elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
                        _error = ServiceError(self.servicename, msg.message, msg.code)
        finally:
            self.pipe.settimeout(0) #return to non-blocking mode

    def _on_message(self, args):
        msg = Message.initialize(args)
        if msg is None:
            return
        try:
            if msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"):
                self._subscribers[msg.session][0](unpackb(msg.data))
            elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"):
                self._subscribers.pop(msg.session, None)
            elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"):
                self._subscribers[msg.session][1](ServiceError(self.servicename, msg.message, msg.code))
        except Exception as err:
            print "Exception in _on_message: %s" % str(err)
コード例 #12
0
class Worker(object):

    def __init__(self, init_args=None, disown_timeout=2, heartbeat_timeout=20):
        self._logger = core_log
        self._init_endpoint(init_args or sys.argv)

        self.sessions = dict()
        self.sandbox = Sandbox()

        self.loop = ev.Loop()

        self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop)
        self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop)
        self.heartbeat_timer.start()

        if isinstance(self.endpoint, types.TupleType) or isinstance(self.endpoint, types.ListType):
            if len(self.endpoint) == 2:
                socket_type = socket.AF_INET
            elif len(self.endpoint) == 4:
                socket_type = socket.AF_INET6
            else:
                raise ValueError('invalid endpoint')
        elif isinstance(self.endpoint, types.StringType):
            socket_type = socket.AF_UNIX
        else:
            raise ValueError('invalid endpoint')
        sock = socket.socket(socket_type)
        self.pipe = Pipe(sock)
        self.pipe.connect(self.endpoint, blocking=True)
        self.loop.bind_on_fd(self.pipe.fileno())

        self.decoder = Decoder()
        self.decoder.bind(self.on_message)

        self.w_stream = WritableStream(self.loop, self.pipe)
        self.r_stream = ReadableStream(self.loop, self.pipe)
        self.r_stream.bind(self.decoder.decode)

        self.loop.register_read_event(self.r_stream._on_event,
                                      self.pipe.fileno())
        self._logger.debug("Worker with %s send handshake" % self.id)
        # Send both messages - to run timers properly. This messages will be sent
        # only after all initialization, so they have same purpose.
        self._send_handshake()
        self._send_heartbeat()

    def _init_endpoint(self, init_args):
        try:
            self.id = init_args[init_args.index("--uuid") + 1]
            # app_name = init_args[init_args.index("--app") + 1]
            self.endpoint = init_args[init_args.index("--endpoint") + 1]
        except Exception as err:
            self._logger.error("Wrong cmdline arguments: %s " % err)
            raise RuntimeError("Wrong cmdline arguments")

    def run(self, binds=None):
        if not binds:
            binds = {}
        for event, name in binds.iteritems():
            self.on(event, name)
        self.loop.run()

    def terminate(self, reason, msg):
        self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack())
        self.loop.stop()
        exit(1)

    # Event machine
    def on(self, event, callback):
        self.sandbox.on(event, callback)

    # Events
    def on_heartbeat(self):
        self._send_heartbeat()

    def on_message(self, args):
        msg = Message.initialize(args)
        if msg is None:
            return

        elif msg.id == message.RPC_INVOKE:
            request = Request()
            stream = Stream(msg.session, self, msg.event)
            try:
                self.sandbox.invoke(msg.event, request, stream)
                self.sessions[msg.session] = request
            except (ImportError, SyntaxError) as err:
                stream.error(2, "unrecoverable error: %s " % str(err))
                self.terminate(1, "Bad code")
            except Exception as err:
                self._logger.error("On invoke error: %s" % err)
                traceback.print_stack()
                stream.error(1, "Invocation error")

        elif msg.id == message.RPC_CHUNK:
            self._logger.debug("Receive chunk: %d" % msg.session)
            try:
                _session = self.sessions[msg.session]
                _session.push(msg.data)
            except Exception as err:
                self._logger.error("On push error: %s" % str(err))
                self.terminate(1, "Push error: %s" % str(err))
                return

        elif msg.id == message.RPC_CHOKE:
            self._logger.debug("Receive choke: %d" % msg.session)
            _session = self.sessions.get(msg.session, None)
            if _session is not None:
                _session.close()
                self.sessions.pop(msg.session)

        elif msg.id == message.RPC_HEARTBEAT:
            self._logger.debug("Receive heartbeat. Stop disown timer")
            self.disown_timer.stop()

        elif msg.id == message.RPC_TERMINATE:
            self._logger.debug("Receive terminate. %s, %s" % (msg.reason, msg.message))
            self.terminate(msg.reason, msg.message)

        elif msg.id == message.RPC_ERROR:
            _session = self.sessions.get(msg.session, None)
            if _session is not None:
                _session.error(RequestError(msg.message))

    def on_disown(self):
        try:
            self._logger.error("Disowned")
        finally:
            self.loop.stop()

    # Private:
    def _send_handshake(self):
        self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack())

    def _send_heartbeat(self):
        self.disown_timer.start()
        self._logger.debug("Send heartbeat. Start disown timer")
        self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack())

    def send_choke(self, session):
        self.w_stream.write(Message(message.RPC_CHOKE, session).pack())

    def send_chunk(self, session, data):
        self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())

    def send_error(self, session, code, msg):
        self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())