Exemplo n.º 1
0
    def _get_master_id_from_node(self, node_id):
        command = protocol.WhoMaster()
        data = ''.join(command.serialize())

        connection = self._send_message(node_id, data)

        receiver = command.receive()
        return utils.read_blocking(receiver, connection.read)
Exemplo n.º 2
0
    def _get_master_id_from_node(self, node_id):
        command = protocol.WhoMaster()
        data = ''.join(command.serialize())

        connection = self._send_message(node_id, data)

        receiver = command.receive()
        return utils.read_blocking(receiver, connection.read)
Exemplo n.º 3
0
def parse_nursery_routing(read):
    '''Parse nursery routing information from a given stream

    :param read: Function returning the requested amount of data
    :type read: `callable` of `int -> str`
    '''

    is_leaf = utils.read_blocking(protocol.BOOL.receive(), read)

    if is_leaf:
        cluster = utils.read_blocking(protocol.STRING.receive(), read)

        return LeafNode(cluster)
    else:
        boundary = utils.read_blocking(protocol.STRING.receive(), read)

        left = parse_nursery_routing(read)
        right = parse_nursery_routing(read)

        return InternalNode(boundary, left, right)
Exemplo n.º 4
0
    def receive(self):
        buffer_receiver = protocol.STRING.receive()
        request = buffer_receiver.next() #pylint: disable=E1101

        while isinstance(request, protocol.Request):
            value = yield request
            request = buffer_receiver.send(value) #pylint: disable=E1101

        if not isinstance(request, protocol.Result):
            raise TypeError

        read = StringIO.StringIO(request.value).read

        routing = parse_nursery_routing(read)

        config_count = utils.read_blocking(protocol.UINT32.receive(), read)
        configs = {}

        for _ in xrange(config_count):
            cluster_id = utils.read_blocking(protocol.STRING.receive(), read)
            cluster_size = utils.read_blocking(protocol.UINT32.receive(), read)

            config = {}

            for _ in xrange(cluster_size):
                node_id = utils.read_blocking(protocol.STRING.receive(), read)
                ips = utils.read_blocking(
                    protocol.List(protocol.STRING).receive(), read)
                port = utils.read_blocking(protocol.UINT32.receive(), read)

                config[node_id] = (tuple(ips), port)

            configs[cluster_id] = config

        yield protocol.Result(NurseryConfig(routing, configs))
Exemplo n.º 5
0
def test_read_blocking():
    '''Test `read_blocking`'''

    data = StringIO.StringIO(''.join(chr(c) for c in (
        0, 0, 0, 0, 3, 0, 0, 0, ord('x'), ord('x'), ord('x'))))

    result = utils.read_blocking(protocol.Hello('testsuite',
        'pyrakoon_test').receive(), data.read)

    if HAS_NOSE:
        nose.tools.assert_equals(result, 'xxx')
    else:
        assert result == 'xxx'
Exemplo n.º 6
0
def test_read_blocking():
    '''Test `read_blocking`'''

    data = StringIO.StringIO(''.join(
        chr(c)
        for c in (0, 0, 0, 0, 3, 0, 0, 0, ord('x'), ord('x'), ord('x'))))

    result = utils.read_blocking(
        protocol.Hello('testsuite', 'pyrakoon_test').receive(), data.read)

    if HAS_NOSE:
        nose.tools.assert_equals(result, 'xxx')
    else:
        assert result == 'xxx'
Exemplo n.º 7
0
    def _process(self, message, node_id = None, retry = True):
        
        bytes_ = ''.join(message.serialize())

        self._lock.acquire()

        try:
            start = time.time()
            tryCount = 0.0
            backoffPeriod = 0.2
            retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
            deadline = start + retryPeriod
            while True:
                try:
                    # Send on wire
                    if node_id is None:
                        connection = self._send_to_master(bytes_)
                    else:
                        connection = self._send_message(node_id, bytes_)
                    return utils.read_blocking(message.receive(),
                        connection.read)
                except (errors.NotMaster,
                        ArakoonNoMaster,
                        ArakoonNotConnected,
                        ArakoonSockReadNoBytes):
                    self.master_id = None
                    self.drop_connections()
                    
                    sleepPeriod = backoffPeriod * tryCount
                    if retry and time.time() + sleepPeriod <= deadline:
                        tryCount += 1.0
                        LOGGER.warning(
                            'Master not found, retrying in %0.2f seconds' % \
                            sleepPeriod)
                        time.sleep(sleepPeriod)
                    else:
                        raise

        finally:
            self._lock.release()
Exemplo n.º 8
0
    def _process(self, message):
        bytes_ = ''.join(message.serialize())

        self._lock.acquire()

        try:
            start = time.time()
            tryCount = 0.0
            backoffPeriod = 0.2
            callSucceeded = False
            retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
            deadline = start + retryPeriod

            while not callSucceeded and time.time() < deadline:
                try:
                    # Send on wire
                    connection = self._send_to_master(bytes_)
                    return utils.read_blocking(message.receive(),
                                               connection.read)
                except (errors.NotMaster, ArakoonNoMaster):
                    self.master_id = None
                    self.drop_connections()

                    sleepPeriod = backoffPeriod * tryCount
                    if time.time() + sleepPeriod > deadline:
                        raise

                    tryCount += 1.0
                    LOGGER.warning(
                        'Master not found, retrying in %0.2f seconds' % \
                            sleepPeriod)

                    time.sleep(sleepPeriod)

        finally:
            self._lock.release()
Exemplo n.º 9
0
    def _process(self, message):
        bytes_ = ''.join(message.serialize())

        self._lock.acquire()

        try:
            start = time.time()
            tryCount = 0.0
            backoffPeriod = 0.2
            callSucceeded = False
            retryPeriod = ArakoonClientConfig.getNoMasterRetryPeriod()
            deadline = start + retryPeriod

            while not callSucceeded and time.time() < deadline:
                try:
                    # Send on wire
                    connection = self._send_to_master(bytes_)
                    return utils.read_blocking(message.receive(),
                        connection.read)
                except (errors.NotMaster, ArakoonNoMaster):
                    self.master_id = None
                    self.drop_connections()

                    sleepPeriod = backoffPeriod * tryCount
                    if time.time() + sleepPeriod > deadline:
                        raise

                    tryCount += 1.0
                    LOGGER.warning(
                        'Master not found, retrying in %0.2f seconds' % \
                            sleepPeriod)

                    time.sleep(sleepPeriod)

        finally:
            self._lock.release()
Exemplo n.º 10
0
    def receive(self):
        buffer_receiver = STRING.receive()
        request = buffer_receiver.next() #pylint: disable=E1101

        while isinstance(request, Request):
            value = yield request
            request = buffer_receiver.send(value) #pylint: disable=E1101

        if not isinstance(request, Result):
            raise TypeError

        read = StringIO.StringIO(request.value).read

        class NamedField(Type):
            '''NamedField type'''

            FIELD_TYPE_INT = 1
            FIELD_TYPE_INT64 = 2
            FIELD_TYPE_FLOAT = 3
            FIELD_TYPE_STRING = 4
            FIELD_TYPE_LIST = 5

            def check(self, value):
                raise NotImplementedError('NamedFields can\'t be checked')

            def serialize(self, value):
                raise NotImplementedError('NamedFields can\'t be serialized')

            @classmethod
            def receive(cls):
                type_receiver = INT32.receive()
                request = type_receiver.next() #pylint: disable=E1101

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1101
                    request = type_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                type_ = request.value

                name_receiver = STRING.receive()
                request = name_receiver.next() #pylint: disable=E1101

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1101
                    request = name_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                name = request.value

                if type_ == cls.FIELD_TYPE_INT:
                    value_receiver = INT32.receive()
                elif type_ == cls.FIELD_TYPE_INT64:
                    value_receiver = INT64.receive()
                elif type_ == cls.FIELD_TYPE_FLOAT:
                    value_receiver = FLOAT.receive()
                elif type_ == cls.FIELD_TYPE_STRING:
                    value_receiver = STRING.receive()
                elif type_ == cls.FIELD_TYPE_LIST:
                    value_receiver = List(NamedField).receive()
                else:
                    raise ValueError('Unknown named field type %d' % type_)

                request = value_receiver.next() #pylint: disable=E1103

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1103
                    request = value_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                value = request.value

                if type_ == cls.FIELD_TYPE_LIST:
                    result = dict()
                    map(result.update, value) #pylint: disable=W0141
                    value = result

                yield Result({name: value})

        result = utils.read_blocking(NamedField.receive(), read)

        if 'arakoon_stats' not in result:
            raise ValueError('Missing expected \'arakoon_stats\' value')

        yield Result(result['arakoon_stats'])
Exemplo n.º 11
0
    def receive(self):
        buffer_receiver = STRING.receive()
        request = buffer_receiver.next()  #pylint: disable=E1101

        while isinstance(request, Request):
            value = yield request
            request = buffer_receiver.send(value)  #pylint: disable=E1101

        if not isinstance(request, Result):
            raise TypeError

        read = StringIO.StringIO(request.value).read

        class NamedField(Type):
            '''NamedField type'''

            FIELD_TYPE_INT = 1
            FIELD_TYPE_INT64 = 2
            FIELD_TYPE_FLOAT = 3
            FIELD_TYPE_STRING = 4
            FIELD_TYPE_LIST = 5

            def check(self, value):
                raise NotImplementedError('NamedFields can\'t be checked')

            def serialize(self, value):
                raise NotImplementedError('NamedFields can\'t be serialized')

            @classmethod
            def receive(cls):
                type_receiver = INT32.receive()
                request = type_receiver.next()  #pylint: disable=E1101

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1101
                    request = type_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                type_ = request.value

                name_receiver = STRING.receive()
                request = name_receiver.next()  #pylint: disable=E1101

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1101
                    request = name_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                name = request.value

                if type_ == cls.FIELD_TYPE_INT:
                    value_receiver = INT32.receive()
                elif type_ == cls.FIELD_TYPE_INT64:
                    value_receiver = INT64.receive()
                elif type_ == cls.FIELD_TYPE_FLOAT:
                    value_receiver = FLOAT.receive()
                elif type_ == cls.FIELD_TYPE_STRING:
                    value_receiver = STRING.receive()
                elif type_ == cls.FIELD_TYPE_LIST:
                    value_receiver = List(NamedField).receive()
                else:
                    raise ValueError('Unknown named field type %d' % type_)

                request = value_receiver.next()  #pylint: disable=E1103

                while isinstance(request, Request):
                    value = yield request
                    #pylint: disable=E1103
                    request = value_receiver.send(value)

                if not isinstance(request, Result):
                    raise TypeError

                value = request.value

                if type_ == cls.FIELD_TYPE_LIST:
                    result = dict()
                    map(result.update, value)  #pylint: disable=W0141
                    value = result

                yield Result({name: value})

        result = utils.read_blocking(NamedField.receive(), read)

        if 'arakoon_stats' not in result:
            raise ValueError('Missing expected \'arakoon_stats\' value')

        yield Result(result['arakoon_stats'])
Exemplo n.º 12
0
    def _process(self, message): #pylint: disable=R0912
        bytes_ = StringIO.StringIO(''.join(message.serialize())).read

        # Helper
        recv = lambda type_: utils.read_blocking(type_.receive(), bytes_)

        command = recv(protocol.UINT32)

        def handle_hello():
            '''Handle a "hello" command'''

            _ = recv(protocol.STRING)
            _ = recv(protocol.STRING)

            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes
            for rbytes in protocol.STRING.serialize(self.VERSION):
                yield rbytes

        def handle_exists():
            '''Handle an "exists" command'''

            _ = recv(protocol.BOOL)
            key = recv(protocol.STRING)

            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes
            for rbytes in protocol.BOOL.serialize(key in self._values):
                yield rbytes

        def handle_who_master():
            '''Handle a "who_master" command'''

            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes
            for rbytes in protocol.Option(protocol.STRING).serialize(
                self.MASTER):
                yield rbytes

        def handle_get():
            '''Handle a "get" command'''

            _ = recv(protocol.BOOL)
            key = recv(protocol.STRING)

            if key not in self._values:
                for rbytes in protocol.UINT32.serialize(
                    errors.NotFound.CODE):
                    yield rbytes
                for rbytes in protocol.STRING.serialize(key):
                    yield rbytes
            else:
                for rbytes in protocol.UINT32.serialize(
                    protocol.RESULT_SUCCESS):
                    yield rbytes
                for rbytes in protocol.STRING.serialize(self._values[key]):
                    yield rbytes

        def handle_set():
            '''Handle a "set" command'''

            key = recv(protocol.STRING)
            value = recv(protocol.STRING)

            self._values[key] = value

            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes

        def handle_delete():
            '''Handle a "delete" command'''

            key = recv(protocol.STRING)

            if key not in self._values:
                for rbytes in protocol.UINT32.serialize(
                    errors.NotFound.CODE):
                    yield rbytes
                for rbytes in protocol.STRING.serialize(key):
                    yield rbytes
            else:
                del self._values[key]
                for rbytes in protocol.UINT32.serialize(
                    protocol.RESULT_SUCCESS):
                    yield rbytes

        def handle_prefix_keys():
            '''Handle a "prefix_keys" command'''

            _ = recv(protocol.BOOL)
            prefix = recv(protocol.STRING)
            max_elements = recv(protocol.UINT32)

            matches = [key for key in self._values.iterkeys()
                if key.startswith(prefix)]

            matches = matches if max_elements < 0 else matches[:max_elements]

            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes

            for rbytes in protocol.List(protocol.STRING).serialize(matches):
                yield rbytes

        def handle_test_and_set():
            '''Handle a "test_and_set" command'''

            key = recv(protocol.STRING)
            test_value = recv(protocol.Option(protocol.STRING))
            set_value = recv(protocol.Option(protocol.STRING))

            # Key doesn't exist and test_value is not None -> NotFound
            if key not in self._values and test_value is not None:
                for rbytes in protocol.UINT32.serialize(
                    errors.NotFound.CODE):
                    yield rbytes
                for rbytes in protocol.STRING.serialize(key):
                    yield rbytes

                return

            # Key doesn't exist and test_value is None -> create
            if key not in self._values and test_value is None:
                self._values[key] = set_value

                for rbytes in protocol.UINT32.serialize(
                    protocol.RESULT_SUCCESS):
                    yield rbytes
                for rbytes in protocol.Option(protocol.STRING).serialize(None):
                    yield rbytes

                return

            # Key exists
            orig_value = self._values[key]

            # Need to update?
            if test_value == orig_value:
                if set_value is not None:
                    self._values[key] = set_value
                else:
                    del self._values[key]

            # Return original value
            for rbytes in protocol.UINT32.serialize(
                protocol.RESULT_SUCCESS):
                yield rbytes

            for rbytes in protocol.Option(protocol.STRING).serialize(
                orig_value):
                yield rbytes


        handlers = {
            protocol.Hello.TAG: handle_hello,
            protocol.Exists.TAG: handle_exists,
            protocol.WhoMaster.TAG: handle_who_master,
            protocol.Get.TAG: handle_get,
            protocol.Set.TAG: handle_set,
            protocol.Delete.TAG: handle_delete,
            protocol.PrefixKeys.TAG: handle_prefix_keys,
            protocol.TestAndSet.TAG: handle_test_and_set,
        }

        if command in handlers:
            result = StringIO.StringIO(''.join(handlers[command]()))
        else:
            result = StringIO.StringIO()
            result.write(struct.pack('<I', errors.UnknownFailure.CODE))
            result.write(struct.pack('<I', 0))
            result.seek(0)

        return utils.read_blocking(message.receive(), result.read)