Пример #1
0
    def setUp(self):
        self.store = '/tmp/store.json'
        self.dest = '/tmp/dbs'
        self.config = gen_test_config()
        if not os.path.exists(self.dest):
            os.mkdir(self.dest)

        self.databases = DatabaseStore(self.config)
        self.default_db_uid = self.databases.index['name_to_uid']['default']
        self._bootstrap_db(self.databases[self.default_db_uid].connector)
        self.handler = Handler(self.databases)
Пример #2
0
    def setUp(self):
        self.store = '/tmp/store.json'
        self.dest = '/tmp/dbs'
        self.config = gen_test_config()
        if not os.path.exists(self.dest):
            os.mkdir(self.dest)

        self.databases = DatabaseStore(self.config)
        self.default_db_uid = self.databases.index['name_to_uid']['default']
        self._bootstrap_db(self.databases[self.default_db_uid].connector)
        self.handler = Handler(self.databases)
Пример #3
0
    def __init__(self, zmq_context, databases, *args, **kwargs):
        threading.Thread.__init__(self)
        self.instructions = {
            WORKER_STATUS: self._status_inst,
            WORKER_HALT: self._stop_inst,
            WORKER_LAST_ACTION: self._last_activity_inst,
        }
        self.uid = uuid.uuid4().hex
        self.zmq_context = zmq_context

        self.state = self.STATES.IDLE

        # Wire backend and remote control sockets
        self.backend_socket = self.zmq_context.socket(zmq.DEALER)
        self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)

        self.databases = databases
        self.handler = Handler(databases)

        self.running = False
        self.last_operation = (None, None)
Пример #4
0
    def __init__(self, zmq_context, databases, *args, **kwargs):
        threading.Thread.__init__(self)
        self.instructions = {
            WORKER_STATUS: self._status_inst,
            WORKER_HALT: self._stop_inst,
            WORKER_LAST_ACTION: self._last_activity_inst,
        }
        self.uid = uuid.uuid4().hex
        self.zmq_context = zmq_context

        self.state = self.STATES.IDLE

        # Wire backend and remote control sockets
        self.backend_socket = self.zmq_context.socket(zmq.DEALER)
        self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)

        self.databases = databases
        self.handler = Handler(databases)

        self.running = False
        self.last_operation = (None, None)
Пример #5
0
class ApiTests(unittest2.TestCase):
    def _bootstrap_db(self, db):
        for val in xrange(9):
            db.put(str(val), str(val + 10))

    def setUp(self):
        self.store = '/tmp/store.json'
        self.dest = '/tmp/dbs'
        self.config = gen_test_config()
        if not os.path.exists(self.dest):
            os.mkdir(self.dest)

        self.databases = DatabaseStore(self.config)
        self.default_db_uid = self.databases.index['name_to_uid']['default']
        self._bootstrap_db(self.databases[self.default_db_uid].connector)
        self.handler = Handler(self.databases)

    def tearDown(self):
        self.databases.__del__()
        del self.handler
        os.remove(self.store)
        shutil.rmtree(self.dest)

    def request_message(self, command, args, db_uid=None):
        db_uid = db_uid or self.default_db_uid
        return Request(
            msgpack.packb({
                'uid': db_uid,
                'cmd': command,
                'args': args,
            }))

    def test_command_with_existing_command(self):
        message = self.request_message('GET', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertNotEqual(plain_content['datas'], None)

    def test_command_with_non_existing_command(self):
        message = self.request_message('COTCOT', ['testarg'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], KEY_ERROR)

    def test_command_with_invalid_db_uid(self):
        message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)

    def test_get_of_existing_key(self):
        message = self.request_message('GET', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], ('11', ))

    def test_get_of_non_existing_key(self):
        message = self.request_message('GET', ['abc123'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], KEY_ERROR)

    def test_mget_of_existing_keys(self):
        message = self.request_message('MGET', [['1', '2', '3']])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], ('11', '12', '13'))

    def test_mget_of_not_fully_existing_keys(self):
        message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], WARNING_STATUS)
        self.assertEqual(len(plain_content['datas']), 3)
        self.assertEqual(plain_content['datas'], ('11', '12', None))

    def test_put_of_valid_key(self):
        message = self.request_message('PUT', ['a', '1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_put_of_invalid_value(self):
        message = self.request_message('PUT', ['a', 1])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], TYPE_ERROR)

    def test_delete(self):
        message = self.request_message('DELETE', ['9'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_exists_of_existing_key(self):
        message = self.request_message('EXISTS', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], True)

    def test_exists_of_non_existing_key_1(self):
        message = self.request_message('EXISTS', ['0'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], False)

    def test_exists_of_non_existing_key_2(self):
        message = self.request_message('EXISTS', ['non_existing'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], False)

    def test_range(self):
        message = self.request_message('RANGE', ['1', '2'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(plain_content['datas'][0], ('1', '11'))
        self.assertEqual(plain_content['datas'][1], ('2', '12'))

    def test_range_with_keys_only(self):
        message = self.request_message('RANGE', ['1', '2', True, False])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)

        self.assertEqual(len(plain_content['datas'][0]), 1)
        self.assertEqual(len(plain_content['datas'][1]), 1)

        self.assertEqual(plain_content['datas'][0], ('1'))
        self.assertEqual(plain_content['datas'][1], ('2'))

    def test_range_of_len_one(self):
        """Should still return a tuple of tuple"""
        message = self.request_message('RANGE', ['1', '1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], (('1', '11'), ))

    def test_slice_with_limit(self):
        message = self.request_message('SLICE', ['1', 3])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content['datas']), 3)
        self.assertEqual(plain_content['datas'][0], ('1', '11'))
        self.assertEqual(plain_content['datas'][1], ('2', '12'))
        self.assertEqual(plain_content['datas'][2], ('3', '13'))

    def test_slice_with_limit_value_of_one(self):
        message = self.request_message('SLICE', ['1', 1])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], (('1', '11'), ))

    def test_batch_with_valid_collection(self):
        message = self.request_message('BATCH',
                                       args=[
                                           [(SIGNAL_BATCH_PUT, 'a', 'a'),
                                            (SIGNAL_BATCH_PUT, 'b', 'b'),
                                            (SIGNAL_BATCH_PUT, 'c', 'c')],
                                       ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_batch_with_invalid_signals(self):
        message = self.request_message('BATCH', [
            [(-5, 'a', 'a'), (-5, 'b', 'b'), (-5, 'c', 'c')],
        ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)

    def test_batch_with_invalid_collection_datas_type(self):
        message = self.request_message('BATCH', [
            [(SIGNAL_BATCH_PUT, 'a', 1), (SIGNAL_BATCH_PUT, 'b', 2),
             (SIGNAL_BATCH_PUT, 'c', 3)],
        ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], TYPE_ERROR)

    def test_connect_to_valid_database(self):
        message = Request(
            msgpack.packb({
                'uid': None,
                'cmd': 'DBCONNECT',
                'args': ['default'],
            }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsNotNone(plain_content)

    def test_connect_to_invalid_database(self):
        message = Request(
            msgpack.packb({
                'uid': None,
                'cmd': 'DBCONNECT',
                'args': ['dadaislikeadad']
            }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_connect_automatically_mounts_and_unmounted_db(self):
        # Unmount by hand the database
        db_uid = self.handler.databases.index['name_to_uid']['default']
        self.handler.databases[
            db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
        self.handler.databases[db_uid].connector = None

        message = Request(
            msgpack.packb({
                'db_uid': None,
                'cmd': 'DBCONNECT',
                'args': ['default']
            }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(self.handler.databases[db_uid].status,
                         self.handler.databases.STATUSES.MOUNTED)
        self.assertIsInstance(self.handler.databases[db_uid].connector,
                              plyvel.DB)

    def test_create_valid_db(self):
        message = self.request_message('DBCREATE', ['testdb'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_create_already_existing_db(self):
        message = self.request_message('DBCREATE', ['default'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_drop_valid_db(self):
        message = self.request_message('DBDROP', ['default'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

        # Please teardown with something to destroy
        # MOUAHAHAHAH... Hum sorry.
        os.mkdir('/tmp/default')

    def test_drop_non_existing_db(self):
        message = self.request_message('DBDROP', ['testdb'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_list_db(self):
        message = self.request_message('DBLIST', [])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], ('default', ))
Пример #6
0
class Worker(threading.Thread):
    STATES = enum('PROCESSING', 'IDLE', 'STOPPED')

    def __init__(self, zmq_context, databases, *args, **kwargs):
        threading.Thread.__init__(self)
        self.instructions = {
            WORKER_STATUS: self._status_inst,
            WORKER_HALT: self._stop_inst,
            WORKER_LAST_ACTION: self._last_activity_inst,
        }
        self.uid = uuid.uuid4().hex
        self.zmq_context = zmq_context

        self.state = self.STATES.IDLE

        # Wire backend and remote control sockets
        self.backend_socket = self.zmq_context.socket(zmq.DEALER)
        self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)

        self.databases = databases
        self.handler = Handler(databases)

        self.running = False
        self.last_operation = (None, None)

    def wire_sockets(self):
        """Connects the worker sockets to their endpoints, and sends
        alive signal to the supervisor"""
        self.backend_socket.connect('inproc://backend')
        self.remote_control_socket.connect('inproc://supervisor')
        self.remote_control_socket.send_multipart(
            [ServiceMessage.dumps(self.uid)])

    def _status_inst(self):
        return str(self.state)

    def _stop_inst(self):
        return self.stop()

    def _last_activity_inst(self):
        return self.last_operation

    def handle_service_message(self):
        """Handles incoming service messages from supervisor socket"""
        try:
            serialized_request = self.remote_control_socket.recv_multipart(
                flags=zmq.NOBLOCK)[0]
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                return

        instruction = ServiceMessage.loads(serialized_request)[0]

        try:
            response = self.instructions[instruction]()
        except KeyError:
            errors_logger.exception("%s instruction not recognized by worker" %
                                    instruction)
            return

        self.remote_control_socket.send_multipart(
            [ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)

        # If halt instruction succedded, raise HaltException
        # so the worker event loop knows it has to stop
        if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
            raise HaltException()
        return

    def handle_command(self):
        """Handles incoming command messages from backend socket

        Receives incoming messages in a non blocking way,
        sets it's set accordingly to IDLE or PROCESSING,
        and sends the responses in a non-blocking way.
        """
        msg = None
        try:
            sender_id, msg = self.backend_socket.recv_multipart(
                copy=False, flags=zmq.NOBLOCK)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                return
        self.state = self.STATES.PROCESSING

        try:
            message = Request(msg)
        except MessageFormatError as e:
            errors_logger.exception(e.value)
            header = ResponseHeader(status=FAILURE_STATUS,
                                    err_code=REQUEST_ERROR,
                                    err_msg=e.value)
            content = ResponseContent(datas={})
            self.backend_socket.send_multipart([sender_id, header, content],
                                               copy=False)
            return

        # Handle message, and execute the requested
        # command in leveldb
        header, response = self.handler.command(message)
        self.last_operation = (time.time(), message.db_uid)

        self.backend_socket.send_multipart([sender_id, header, response],
                                           flags=zmq.NOBLOCK,
                                           copy=False)
        self.state = self.STATES.IDLE

        return

    def run(self):
        """Non blocking event loop which polls for supervisor
        or backend events"""
        poller = zmq.Poller()
        poller.register(self.backend_socket, zmq.POLLIN)
        poller.register(self.remote_control_socket, zmq.POLLIN)

        # Connect sockets, and send the supervisor
        # alive signals
        self.wire_sockets()

        while (self.state != self.STATES.STOPPED):
            sockets = dict(poller.poll())
            if sockets:
                if sockets.get(self.remote_control_socket) == zmq.POLLIN:
                    try:
                        self.handle_service_message()
                    except HaltException:
                        break

                if sockets.get(self.backend_socket) == zmq.POLLIN:
                    self.handle_command()  # Might change state

    def stop(self):
        """Stops the worker

        Changes it's state to STOPPED.
        Closes it's backend socket.
        Returns SUCCESS_STATUS.
        """
        self.state = self.STATES.STOPPED

        if not self.backend_socket.closed:
            self.backend_socket.close()

        activity_logger.info("Gracefully stopping worker %s" % self.uid)
        return str(SUCCESS_STATUS)
Пример #7
0
class ApiTests(unittest2.TestCase):
    def _bootstrap_db(self, db):
        for val in xrange(9):
            db.put(str(val), str(val + 10))

    def setUp(self):
        self.store = '/tmp/store.json'
        self.dest = '/tmp/dbs'
        self.config = gen_test_config()
        if not os.path.exists(self.dest):
            os.mkdir(self.dest)

        self.databases = DatabaseStore(self.config)
        self.default_db_uid = self.databases.index['name_to_uid']['default']
        self._bootstrap_db(self.databases[self.default_db_uid].connector)
        self.handler = Handler(self.databases)

    def tearDown(self):
        self.databases.__del__()
        del self.handler
        os.remove(self.store)
        shutil.rmtree(self.dest)

    def request_message(self, command, args, db_uid=None):
        db_uid = db_uid or self.default_db_uid
        return Request(msgpack.packb({
            'uid': db_uid,
            'cmd': command,
            'args': args,
        }))

    def test_command_with_existing_command(self):
        message = self.request_message('GET', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertNotEqual(plain_content['datas'], None)

    def test_command_with_non_existing_command(self):
        message = self.request_message('COTCOT', ['testarg'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], KEY_ERROR)

    def test_command_with_invalid_db_uid(self):
        message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)

    def test_get_of_existing_key(self):
        message = self.request_message('GET', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], ('11',))

    def test_get_of_non_existing_key(self):
        message = self.request_message('GET', ['abc123'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], KEY_ERROR)

    def test_mget_of_existing_keys(self):
        message = self.request_message('MGET', [['1', '2', '3']])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], ('11', '12', '13'))

    def test_mget_of_not_fully_existing_keys(self):
        message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], WARNING_STATUS)
        self.assertEqual(len(plain_content['datas']), 3)
        self.assertEqual(plain_content['datas'], ('11', '12', None))

    def test_put_of_valid_key(self):
        message = self.request_message('PUT', ['a', '1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_put_of_invalid_value(self):
        message = self.request_message('PUT', ['a', 1])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], TYPE_ERROR)

    def test_delete(self):
        message = self.request_message('DELETE', ['9'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_exists_of_existing_key(self):
        message = self.request_message('EXISTS', ['1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], True)

    def test_exists_of_non_existing_key_1(self):
        message = self.request_message('EXISTS', ['0'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], False)

    def test_exists_of_non_existing_key_2(self):
        message = self.request_message('EXISTS', ['non_existing'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], False)

    def test_range(self):
        message = self.request_message('RANGE', ['1', '2'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(plain_content['datas'][0], ('1', '11'))
        self.assertEqual(plain_content['datas'][1], ('2', '12'))

    def test_range_with_keys_only(self):
        message = self.request_message('RANGE', ['1', '2', True, False])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)

        self.assertEqual(len(plain_content['datas'][0]), 1)
        self.assertEqual(len(plain_content['datas'][1]), 1)

        self.assertEqual(plain_content['datas'][0], ('1'))
        self.assertEqual(plain_content['datas'][1], ('2'))

    def test_range_of_len_one(self):
        """Should still return a tuple of tuple"""
        message = self.request_message('RANGE', ['1', '1'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], (('1', '11'),))

    def test_slice_with_limit(self):
        message = self.request_message('SLICE', ['1', 3])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content['datas']), 3)
        self.assertEqual(plain_content['datas'][0], ('1', '11'))
        self.assertEqual(plain_content['datas'][1], ('2', '12'))
        self.assertEqual(plain_content['datas'][2], ('3', '13'))

    def test_slice_with_limit_value_of_one(self):
        message = self.request_message('SLICE', ['1', 1])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsInstance(plain_content['datas'], tuple)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], (('1', '11'),))

    def test_batch_with_valid_collection(self):
        message = self.request_message('BATCH', args=[
            [(SIGNAL_BATCH_PUT, 'a', 'a'),
             (SIGNAL_BATCH_PUT, 'b', 'b'),
             (SIGNAL_BATCH_PUT, 'c', 'c')],
        ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_batch_with_invalid_signals(self):
        message = self.request_message('BATCH', [
            [(-5, 'a', 'a'),
             (-5, 'b', 'b'),
             (-5, 'c', 'c')],
        ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)

    def test_batch_with_invalid_collection_datas_type(self):
        message = self.request_message('BATCH', [
            [(SIGNAL_BATCH_PUT, 'a', 1),
             (SIGNAL_BATCH_PUT, 'b', 2),
             (SIGNAL_BATCH_PUT, 'c', 3)],
        ])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], TYPE_ERROR)

    def test_connect_to_valid_database(self):
        message = Request(msgpack.packb({
            'uid': None,
            'cmd': 'DBCONNECT',
            'args': ['default'],
        }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertIsNotNone(plain_content)

    def test_connect_to_invalid_database(self):
        message = Request(msgpack.packb({
                'uid': None,
                'cmd': 'DBCONNECT',
                'args': ['dadaislikeadad']
        }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_connect_automatically_mounts_and_unmounted_db(self):
        # Unmount by hand the database
        db_uid = self.handler.databases.index['name_to_uid']['default']
        self.handler.databases[db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
        self.handler.databases[db_uid].connector = None

        message = Request(msgpack.packb({
                'db_uid': None,
                'cmd': 'DBCONNECT',
                'args': ['default']
        }))
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(self.handler.databases[db_uid].status, self.handler.databases.STATUSES.MOUNTED)
        self.assertIsInstance(self.handler.databases[db_uid].connector, plyvel.DB)

    def test_create_valid_db(self):
        message = self.request_message('DBCREATE', ['testdb'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

    def test_create_already_existing_db(self):
        message = self.request_message('DBCREATE', ['default'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_drop_valid_db(self):
        message = self.request_message('DBDROP', ['default'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(plain_content['datas'], None)

        # Please teardown with something to destroy
        # MOUAHAHAHAH... Hum sorry.
        os.mkdir('/tmp/default')

    def test_drop_non_existing_db(self):
        message = self.request_message('DBDROP', ['testdb'])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        self.assertEqual(plain_header['status'], FAILURE_STATUS)
        self.assertEqual(plain_header['err_code'], DATABASE_ERROR)

    def test_list_db(self):
        message = self.request_message('DBLIST', [])
        header, content = self.handler.command(message)

        plain_header = msgpack.unpackb(header)
        plain_content = msgpack.unpackb(content)

        self.assertEqual(plain_header['status'], SUCCESS_STATUS)
        self.assertEqual(len(plain_content), 1)
        self.assertEqual(plain_content['datas'], ('default',))
Пример #8
0
class Worker(threading.Thread):
    STATES = enum("PROCESSING", "IDLE", "STOPPED")

    def __init__(self, zmq_context, databases, *args, **kwargs):
        threading.Thread.__init__(self)
        self.instructions = {
            WORKER_STATUS: self._status_inst,
            WORKER_HALT: self._stop_inst,
            WORKER_LAST_ACTION: self._last_activity_inst,
        }
        self.uid = uuid.uuid4().hex
        self.zmq_context = zmq_context

        self.state = self.STATES.IDLE

        # Wire backend and remote control sockets
        self.backend_socket = self.zmq_context.socket(zmq.DEALER)
        self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)

        self.databases = databases
        self.handler = Handler(databases)

        self.running = False
        self.last_operation = (None, None)

    def wire_sockets(self):
        """Connects the worker sockets to their endpoints, and sends
        alive signal to the supervisor"""
        self.backend_socket.connect("inproc://backend")
        self.remote_control_socket.connect("inproc://supervisor")
        self.remote_control_socket.send_multipart([ServiceMessage.dumps(self.uid)])

    def _status_inst(self):
        return str(self.state)

    def _stop_inst(self):
        return self.stop()

    def _last_activity_inst(self):
        return self.last_operation

    def handle_service_message(self):
        """Handles incoming service messages from supervisor socket"""
        try:
            serialized_request = self.remote_control_socket.recv_multipart(flags=zmq.NOBLOCK)[0]
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                return

        instruction = ServiceMessage.loads(serialized_request)[0]

        try:
            response = self.instructions[instruction]()
        except KeyError:
            errors_logger.exception("%s instruction not recognized by worker" % instruction)
            return

        self.remote_control_socket.send_multipart([ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)

        # If halt instruction succedded, raise HaltException
        # so the worker event loop knows it has to stop
        if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
            raise HaltException()
        return

    def handle_command(self):
        """Handles incoming command messages from backend socket

        Receives incoming messages in a non blocking way,
        sets it's set accordingly to IDLE or PROCESSING,
        and sends the responses in a non-blocking way.
        """
        msg = None
        try:
            sender_id, msg = self.backend_socket.recv_multipart(copy=False, flags=zmq.NOBLOCK)
        except zmq.ZMQError as e:
            if e.errno == zmq.EAGAIN:
                return
        self.state = self.STATES.PROCESSING

        try:
            message = Request(msg)
        except MessageFormatError as e:
            errors_logger.exception(e.value)
            header = ResponseHeader(status=FAILURE_STATUS, err_code=REQUEST_ERROR, err_msg=e.value)
            content = ResponseContent(datas={})
            self.backend_socket.send_multipart([sender_id, header, content], copy=False)
            return

        # Handle message, and execute the requested
        # command in leveldb
        header, response = self.handler.command(message)
        self.last_operation = (time.time(), message.db_uid)

        self.backend_socket.send_multipart([sender_id, header, response], flags=zmq.NOBLOCK, copy=False)
        self.state = self.STATES.IDLE

        return

    def run(self):
        """Non blocking event loop which polls for supervisor
        or backend events"""
        poller = zmq.Poller()
        poller.register(self.backend_socket, zmq.POLLIN)
        poller.register(self.remote_control_socket, zmq.POLLIN)

        # Connect sockets, and send the supervisor
        # alive signals
        self.wire_sockets()

        while self.state != self.STATES.STOPPED:
            sockets = dict(poller.poll())
            if sockets:
                if sockets.get(self.remote_control_socket) == zmq.POLLIN:
                    try:
                        self.handle_service_message()
                    except HaltException:
                        break

                if sockets.get(self.backend_socket) == zmq.POLLIN:
                    self.handle_command()  # Might change state

    def stop(self):
        """Stops the worker

        Changes it's state to STOPPED.
        Closes it's backend socket.
        Returns SUCCESS_STATUS.
        """
        self.state = self.STATES.STOPPED

        if not self.backend_socket.closed:
            self.backend_socket.close()

        activity_logger.info("Gracefully stopping worker %s" % self.uid)
        return str(SUCCESS_STATUS)