def setUp(self): self.store = '/tmp/store.json' self.dest = '/tmp/dbs' self.config = gen_test_config() if not os.path.exists(self.dest): os.mkdir(self.dest) self.databases = DatabaseStore(self.config) self.default_db_uid = self.databases.index['name_to_uid']['default'] self._bootstrap_db(self.databases[self.default_db_uid].connector) self.handler = Handler(self.databases)
def __init__(self, zmq_context, databases, *args, **kwargs): threading.Thread.__init__(self) self.instructions = { WORKER_STATUS: self._status_inst, WORKER_HALT: self._stop_inst, WORKER_LAST_ACTION: self._last_activity_inst, } self.uid = uuid.uuid4().hex self.zmq_context = zmq_context self.state = self.STATES.IDLE # Wire backend and remote control sockets self.backend_socket = self.zmq_context.socket(zmq.DEALER) self.remote_control_socket = self.zmq_context.socket(zmq.DEALER) self.databases = databases self.handler = Handler(databases) self.running = False self.last_operation = (None, None)
class ApiTests(unittest2.TestCase): def _bootstrap_db(self, db): for val in xrange(9): db.put(str(val), str(val + 10)) def setUp(self): self.store = '/tmp/store.json' self.dest = '/tmp/dbs' self.config = gen_test_config() if not os.path.exists(self.dest): os.mkdir(self.dest) self.databases = DatabaseStore(self.config) self.default_db_uid = self.databases.index['name_to_uid']['default'] self._bootstrap_db(self.databases[self.default_db_uid].connector) self.handler = Handler(self.databases) def tearDown(self): self.databases.__del__() del self.handler os.remove(self.store) shutil.rmtree(self.dest) def request_message(self, command, args, db_uid=None): db_uid = db_uid or self.default_db_uid return Request( msgpack.packb({ 'uid': db_uid, 'cmd': command, 'args': args, })) def test_command_with_existing_command(self): message = self.request_message('GET', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertNotEqual(plain_content['datas'], None) def test_command_with_non_existing_command(self): message = self.request_message('COTCOT', ['testarg']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], KEY_ERROR) def test_command_with_invalid_db_uid(self): message = self.request_message('PUT', ['1', '1'], db_uid='failinguid') header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], RUNTIME_ERROR) def test_get_of_existing_key(self): message = self.request_message('GET', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], ('11', )) def test_get_of_non_existing_key(self): message = self.request_message('GET', ['abc123']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], KEY_ERROR) def test_mget_of_existing_keys(self): message = self.request_message('MGET', [['1', '2', '3']]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], ('11', '12', '13')) def test_mget_of_not_fully_existing_keys(self): message = self.request_message('MGET', [['1', '2', 'touptoupidou']]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], WARNING_STATUS) self.assertEqual(len(plain_content['datas']), 3) self.assertEqual(plain_content['datas'], ('11', '12', None)) def test_put_of_valid_key(self): message = self.request_message('PUT', ['a', '1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_put_of_invalid_value(self): message = self.request_message('PUT', ['a', 1]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], TYPE_ERROR) def test_delete(self): message = self.request_message('DELETE', ['9']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_exists_of_existing_key(self): message = self.request_message('EXISTS', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], True) def test_exists_of_non_existing_key_1(self): message = self.request_message('EXISTS', ['0']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], False) def test_exists_of_non_existing_key_2(self): message = self.request_message('EXISTS', ['non_existing']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], False) def test_range(self): message = self.request_message('RANGE', ['1', '2']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(plain_content['datas'][0], ('1', '11')) self.assertEqual(plain_content['datas'][1], ('2', '12')) def test_range_with_keys_only(self): message = self.request_message('RANGE', ['1', '2', True, False]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content['datas'][0]), 1) self.assertEqual(len(plain_content['datas'][1]), 1) self.assertEqual(plain_content['datas'][0], ('1')) self.assertEqual(plain_content['datas'][1], ('2')) def test_range_of_len_one(self): """Should still return a tuple of tuple""" message = self.request_message('RANGE', ['1', '1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], (('1', '11'), )) def test_slice_with_limit(self): message = self.request_message('SLICE', ['1', 3]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content['datas']), 3) self.assertEqual(plain_content['datas'][0], ('1', '11')) self.assertEqual(plain_content['datas'][1], ('2', '12')) self.assertEqual(plain_content['datas'][2], ('3', '13')) def test_slice_with_limit_value_of_one(self): message = self.request_message('SLICE', ['1', 1]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], (('1', '11'), )) def test_batch_with_valid_collection(self): message = self.request_message('BATCH', args=[ [(SIGNAL_BATCH_PUT, 'a', 'a'), (SIGNAL_BATCH_PUT, 'b', 'b'), (SIGNAL_BATCH_PUT, 'c', 'c')], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_batch_with_invalid_signals(self): message = self.request_message('BATCH', [ [(-5, 'a', 'a'), (-5, 'b', 'b'), (-5, 'c', 'c')], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], SIGNAL_ERROR) def test_batch_with_invalid_collection_datas_type(self): message = self.request_message('BATCH', [ [(SIGNAL_BATCH_PUT, 'a', 1), (SIGNAL_BATCH_PUT, 'b', 2), (SIGNAL_BATCH_PUT, 'c', 3)], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], TYPE_ERROR) def test_connect_to_valid_database(self): message = Request( msgpack.packb({ 'uid': None, 'cmd': 'DBCONNECT', 'args': ['default'], })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsNotNone(plain_content) def test_connect_to_invalid_database(self): message = Request( msgpack.packb({ 'uid': None, 'cmd': 'DBCONNECT', 'args': ['dadaislikeadad'] })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_connect_automatically_mounts_and_unmounted_db(self): # Unmount by hand the database db_uid = self.handler.databases.index['name_to_uid']['default'] self.handler.databases[ db_uid].status = self.handler.databases.STATUSES.UNMOUNTED self.handler.databases[db_uid].connector = None message = Request( msgpack.packb({ 'db_uid': None, 'cmd': 'DBCONNECT', 'args': ['default'] })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(self.handler.databases[db_uid].status, self.handler.databases.STATUSES.MOUNTED) self.assertIsInstance(self.handler.databases[db_uid].connector, plyvel.DB) def test_create_valid_db(self): message = self.request_message('DBCREATE', ['testdb']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_create_already_existing_db(self): message = self.request_message('DBCREATE', ['default']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_drop_valid_db(self): message = self.request_message('DBDROP', ['default']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) # Please teardown with something to destroy # MOUAHAHAHAH... Hum sorry. os.mkdir('/tmp/default') def test_drop_non_existing_db(self): message = self.request_message('DBDROP', ['testdb']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_list_db(self): message = self.request_message('DBLIST', []) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], ('default', ))
class Worker(threading.Thread): STATES = enum('PROCESSING', 'IDLE', 'STOPPED') def __init__(self, zmq_context, databases, *args, **kwargs): threading.Thread.__init__(self) self.instructions = { WORKER_STATUS: self._status_inst, WORKER_HALT: self._stop_inst, WORKER_LAST_ACTION: self._last_activity_inst, } self.uid = uuid.uuid4().hex self.zmq_context = zmq_context self.state = self.STATES.IDLE # Wire backend and remote control sockets self.backend_socket = self.zmq_context.socket(zmq.DEALER) self.remote_control_socket = self.zmq_context.socket(zmq.DEALER) self.databases = databases self.handler = Handler(databases) self.running = False self.last_operation = (None, None) def wire_sockets(self): """Connects the worker sockets to their endpoints, and sends alive signal to the supervisor""" self.backend_socket.connect('inproc://backend') self.remote_control_socket.connect('inproc://supervisor') self.remote_control_socket.send_multipart( [ServiceMessage.dumps(self.uid)]) def _status_inst(self): return str(self.state) def _stop_inst(self): return self.stop() def _last_activity_inst(self): return self.last_operation def handle_service_message(self): """Handles incoming service messages from supervisor socket""" try: serialized_request = self.remote_control_socket.recv_multipart( flags=zmq.NOBLOCK)[0] except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return instruction = ServiceMessage.loads(serialized_request)[0] try: response = self.instructions[instruction]() except KeyError: errors_logger.exception("%s instruction not recognized by worker" % instruction) return self.remote_control_socket.send_multipart( [ServiceMessage.dumps(response)], flags=zmq.NOBLOCK) # If halt instruction succedded, raise HaltException # so the worker event loop knows it has to stop if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS: raise HaltException() return def handle_command(self): """Handles incoming command messages from backend socket Receives incoming messages in a non blocking way, sets it's set accordingly to IDLE or PROCESSING, and sends the responses in a non-blocking way. """ msg = None try: sender_id, msg = self.backend_socket.recv_multipart( copy=False, flags=zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return self.state = self.STATES.PROCESSING try: message = Request(msg) except MessageFormatError as e: errors_logger.exception(e.value) header = ResponseHeader(status=FAILURE_STATUS, err_code=REQUEST_ERROR, err_msg=e.value) content = ResponseContent(datas={}) self.backend_socket.send_multipart([sender_id, header, content], copy=False) return # Handle message, and execute the requested # command in leveldb header, response = self.handler.command(message) self.last_operation = (time.time(), message.db_uid) self.backend_socket.send_multipart([sender_id, header, response], flags=zmq.NOBLOCK, copy=False) self.state = self.STATES.IDLE return def run(self): """Non blocking event loop which polls for supervisor or backend events""" poller = zmq.Poller() poller.register(self.backend_socket, zmq.POLLIN) poller.register(self.remote_control_socket, zmq.POLLIN) # Connect sockets, and send the supervisor # alive signals self.wire_sockets() while (self.state != self.STATES.STOPPED): sockets = dict(poller.poll()) if sockets: if sockets.get(self.remote_control_socket) == zmq.POLLIN: try: self.handle_service_message() except HaltException: break if sockets.get(self.backend_socket) == zmq.POLLIN: self.handle_command() # Might change state def stop(self): """Stops the worker Changes it's state to STOPPED. Closes it's backend socket. Returns SUCCESS_STATUS. """ self.state = self.STATES.STOPPED if not self.backend_socket.closed: self.backend_socket.close() activity_logger.info("Gracefully stopping worker %s" % self.uid) return str(SUCCESS_STATUS)
class ApiTests(unittest2.TestCase): def _bootstrap_db(self, db): for val in xrange(9): db.put(str(val), str(val + 10)) def setUp(self): self.store = '/tmp/store.json' self.dest = '/tmp/dbs' self.config = gen_test_config() if not os.path.exists(self.dest): os.mkdir(self.dest) self.databases = DatabaseStore(self.config) self.default_db_uid = self.databases.index['name_to_uid']['default'] self._bootstrap_db(self.databases[self.default_db_uid].connector) self.handler = Handler(self.databases) def tearDown(self): self.databases.__del__() del self.handler os.remove(self.store) shutil.rmtree(self.dest) def request_message(self, command, args, db_uid=None): db_uid = db_uid or self.default_db_uid return Request(msgpack.packb({ 'uid': db_uid, 'cmd': command, 'args': args, })) def test_command_with_existing_command(self): message = self.request_message('GET', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertNotEqual(plain_content['datas'], None) def test_command_with_non_existing_command(self): message = self.request_message('COTCOT', ['testarg']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], KEY_ERROR) def test_command_with_invalid_db_uid(self): message = self.request_message('PUT', ['1', '1'], db_uid='failinguid') header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], RUNTIME_ERROR) def test_get_of_existing_key(self): message = self.request_message('GET', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], ('11',)) def test_get_of_non_existing_key(self): message = self.request_message('GET', ['abc123']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], KEY_ERROR) def test_mget_of_existing_keys(self): message = self.request_message('MGET', [['1', '2', '3']]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], ('11', '12', '13')) def test_mget_of_not_fully_existing_keys(self): message = self.request_message('MGET', [['1', '2', 'touptoupidou']]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], WARNING_STATUS) self.assertEqual(len(plain_content['datas']), 3) self.assertEqual(plain_content['datas'], ('11', '12', None)) def test_put_of_valid_key(self): message = self.request_message('PUT', ['a', '1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_put_of_invalid_value(self): message = self.request_message('PUT', ['a', 1]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], TYPE_ERROR) def test_delete(self): message = self.request_message('DELETE', ['9']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_exists_of_existing_key(self): message = self.request_message('EXISTS', ['1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], True) def test_exists_of_non_existing_key_1(self): message = self.request_message('EXISTS', ['0']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], False) def test_exists_of_non_existing_key_2(self): message = self.request_message('EXISTS', ['non_existing']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], False) def test_range(self): message = self.request_message('RANGE', ['1', '2']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(plain_content['datas'][0], ('1', '11')) self.assertEqual(plain_content['datas'][1], ('2', '12')) def test_range_with_keys_only(self): message = self.request_message('RANGE', ['1', '2', True, False]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content['datas'][0]), 1) self.assertEqual(len(plain_content['datas'][1]), 1) self.assertEqual(plain_content['datas'][0], ('1')) self.assertEqual(plain_content['datas'][1], ('2')) def test_range_of_len_one(self): """Should still return a tuple of tuple""" message = self.request_message('RANGE', ['1', '1']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], (('1', '11'),)) def test_slice_with_limit(self): message = self.request_message('SLICE', ['1', 3]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content['datas']), 3) self.assertEqual(plain_content['datas'][0], ('1', '11')) self.assertEqual(plain_content['datas'][1], ('2', '12')) self.assertEqual(plain_content['datas'][2], ('3', '13')) def test_slice_with_limit_value_of_one(self): message = self.request_message('SLICE', ['1', 1]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsInstance(plain_content['datas'], tuple) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], (('1', '11'),)) def test_batch_with_valid_collection(self): message = self.request_message('BATCH', args=[ [(SIGNAL_BATCH_PUT, 'a', 'a'), (SIGNAL_BATCH_PUT, 'b', 'b'), (SIGNAL_BATCH_PUT, 'c', 'c')], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_batch_with_invalid_signals(self): message = self.request_message('BATCH', [ [(-5, 'a', 'a'), (-5, 'b', 'b'), (-5, 'c', 'c')], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], SIGNAL_ERROR) def test_batch_with_invalid_collection_datas_type(self): message = self.request_message('BATCH', [ [(SIGNAL_BATCH_PUT, 'a', 1), (SIGNAL_BATCH_PUT, 'b', 2), (SIGNAL_BATCH_PUT, 'c', 3)], ]) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], TYPE_ERROR) def test_connect_to_valid_database(self): message = Request(msgpack.packb({ 'uid': None, 'cmd': 'DBCONNECT', 'args': ['default'], })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertIsNotNone(plain_content) def test_connect_to_invalid_database(self): message = Request(msgpack.packb({ 'uid': None, 'cmd': 'DBCONNECT', 'args': ['dadaislikeadad'] })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_connect_automatically_mounts_and_unmounted_db(self): # Unmount by hand the database db_uid = self.handler.databases.index['name_to_uid']['default'] self.handler.databases[db_uid].status = self.handler.databases.STATUSES.UNMOUNTED self.handler.databases[db_uid].connector = None message = Request(msgpack.packb({ 'db_uid': None, 'cmd': 'DBCONNECT', 'args': ['default'] })) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(self.handler.databases[db_uid].status, self.handler.databases.STATUSES.MOUNTED) self.assertIsInstance(self.handler.databases[db_uid].connector, plyvel.DB) def test_create_valid_db(self): message = self.request_message('DBCREATE', ['testdb']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) def test_create_already_existing_db(self): message = self.request_message('DBCREATE', ['default']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_drop_valid_db(self): message = self.request_message('DBDROP', ['default']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(plain_content['datas'], None) # Please teardown with something to destroy # MOUAHAHAHAH... Hum sorry. os.mkdir('/tmp/default') def test_drop_non_existing_db(self): message = self.request_message('DBDROP', ['testdb']) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) self.assertEqual(plain_header['status'], FAILURE_STATUS) self.assertEqual(plain_header['err_code'], DATABASE_ERROR) def test_list_db(self): message = self.request_message('DBLIST', []) header, content = self.handler.command(message) plain_header = msgpack.unpackb(header) plain_content = msgpack.unpackb(content) self.assertEqual(plain_header['status'], SUCCESS_STATUS) self.assertEqual(len(plain_content), 1) self.assertEqual(plain_content['datas'], ('default',))
class Worker(threading.Thread): STATES = enum("PROCESSING", "IDLE", "STOPPED") def __init__(self, zmq_context, databases, *args, **kwargs): threading.Thread.__init__(self) self.instructions = { WORKER_STATUS: self._status_inst, WORKER_HALT: self._stop_inst, WORKER_LAST_ACTION: self._last_activity_inst, } self.uid = uuid.uuid4().hex self.zmq_context = zmq_context self.state = self.STATES.IDLE # Wire backend and remote control sockets self.backend_socket = self.zmq_context.socket(zmq.DEALER) self.remote_control_socket = self.zmq_context.socket(zmq.DEALER) self.databases = databases self.handler = Handler(databases) self.running = False self.last_operation = (None, None) def wire_sockets(self): """Connects the worker sockets to their endpoints, and sends alive signal to the supervisor""" self.backend_socket.connect("inproc://backend") self.remote_control_socket.connect("inproc://supervisor") self.remote_control_socket.send_multipart([ServiceMessage.dumps(self.uid)]) def _status_inst(self): return str(self.state) def _stop_inst(self): return self.stop() def _last_activity_inst(self): return self.last_operation def handle_service_message(self): """Handles incoming service messages from supervisor socket""" try: serialized_request = self.remote_control_socket.recv_multipart(flags=zmq.NOBLOCK)[0] except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return instruction = ServiceMessage.loads(serialized_request)[0] try: response = self.instructions[instruction]() except KeyError: errors_logger.exception("%s instruction not recognized by worker" % instruction) return self.remote_control_socket.send_multipart([ServiceMessage.dumps(response)], flags=zmq.NOBLOCK) # If halt instruction succedded, raise HaltException # so the worker event loop knows it has to stop if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS: raise HaltException() return def handle_command(self): """Handles incoming command messages from backend socket Receives incoming messages in a non blocking way, sets it's set accordingly to IDLE or PROCESSING, and sends the responses in a non-blocking way. """ msg = None try: sender_id, msg = self.backend_socket.recv_multipart(copy=False, flags=zmq.NOBLOCK) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return self.state = self.STATES.PROCESSING try: message = Request(msg) except MessageFormatError as e: errors_logger.exception(e.value) header = ResponseHeader(status=FAILURE_STATUS, err_code=REQUEST_ERROR, err_msg=e.value) content = ResponseContent(datas={}) self.backend_socket.send_multipart([sender_id, header, content], copy=False) return # Handle message, and execute the requested # command in leveldb header, response = self.handler.command(message) self.last_operation = (time.time(), message.db_uid) self.backend_socket.send_multipart([sender_id, header, response], flags=zmq.NOBLOCK, copy=False) self.state = self.STATES.IDLE return def run(self): """Non blocking event loop which polls for supervisor or backend events""" poller = zmq.Poller() poller.register(self.backend_socket, zmq.POLLIN) poller.register(self.remote_control_socket, zmq.POLLIN) # Connect sockets, and send the supervisor # alive signals self.wire_sockets() while self.state != self.STATES.STOPPED: sockets = dict(poller.poll()) if sockets: if sockets.get(self.remote_control_socket) == zmq.POLLIN: try: self.handle_service_message() except HaltException: break if sockets.get(self.backend_socket) == zmq.POLLIN: self.handle_command() # Might change state def stop(self): """Stops the worker Changes it's state to STOPPED. Closes it's backend socket. Returns SUCCESS_STATUS. """ self.state = self.STATES.STOPPED if not self.backend_socket.closed: self.backend_socket.close() activity_logger.info("Gracefully stopping worker %s" % self.uid) return str(SUCCESS_STATUS)