def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def __init__(self, zmq_context, databases, *args, **kwargs):
threading.Thread.__init__(self)
self.instructions = {
WORKER_STATUS: self._status_inst,
WORKER_HALT: self._stop_inst,
WORKER_LAST_ACTION: self._last_activity_inst,
}
self.uid = uuid.uuid4().hex
self.zmq_context = zmq_context
self.state = self.STATES.IDLE
# Wire backend and remote control sockets
self.backend_socket = self.zmq_context.socket(zmq.DEALER)
self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)
self.databases = databases
self.handler = Handler(databases)
self.running = False
self.last_operation = (None, None)
def __init__(self, zmq_context, databases, *args, **kwargs):
threading.Thread.__init__(self)
self.instructions = {
WORKER_STATUS: self._status_inst,
WORKER_HALT: self._stop_inst,
WORKER_LAST_ACTION: self._last_activity_inst,
}
self.uid = uuid.uuid4().hex
self.zmq_context = zmq_context
self.state = self.STATES.IDLE
# Wire backend and remote control sockets
self.backend_socket = self.zmq_context.socket(zmq.DEALER)
self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)
self.databases = databases
self.handler = Handler(databases)
self.running = False
self.last_operation = (None, None)
class ApiTests(unittest2.TestCase):
def _bootstrap_db(self, db):
for val in xrange(9):
db.put(str(val), str(val + 10))
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def tearDown(self):
self.databases.__del__()
del self.handler
os.remove(self.store)
shutil.rmtree(self.dest)
def request_message(self, command, args, db_uid=None):
db_uid = db_uid or self.default_db_uid
return Request(
msgpack.packb({
'uid': db_uid,
'cmd': command,
'args': args,
}))
def test_command_with_existing_command(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertNotEqual(plain_content['datas'], None)
def test_command_with_non_existing_command(self):
message = self.request_message('COTCOT', ['testarg'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_command_with_invalid_db_uid(self):
message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)
def test_get_of_existing_key(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', ))
def test_get_of_non_existing_key(self):
message = self.request_message('GET', ['abc123'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_mget_of_existing_keys(self):
message = self.request_message('MGET', [['1', '2', '3']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', '12', '13'))
def test_mget_of_not_fully_existing_keys(self):
message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], WARNING_STATUS)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'], ('11', '12', None))
def test_put_of_valid_key(self):
message = self.request_message('PUT', ['a', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_put_of_invalid_value(self):
message = self.request_message('PUT', ['a', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_delete(self):
message = self.request_message('DELETE', ['9'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_exists_of_existing_key(self):
message = self.request_message('EXISTS', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], True)
def test_exists_of_non_existing_key_1(self):
message = self.request_message('EXISTS', ['0'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_exists_of_non_existing_key_2(self):
message = self.request_message('EXISTS', ['non_existing'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_range(self):
message = self.request_message('RANGE', ['1', '2'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
def test_range_with_keys_only(self):
message = self.request_message('RANGE', ['1', '2', True, False])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas'][0]), 1)
self.assertEqual(len(plain_content['datas'][1]), 1)
self.assertEqual(plain_content['datas'][0], ('1'))
self.assertEqual(plain_content['datas'][1], ('2'))
def test_range_of_len_one(self):
"""Should still return a tuple of tuple"""
message = self.request_message('RANGE', ['1', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'), ))
def test_slice_with_limit(self):
message = self.request_message('SLICE', ['1', 3])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
self.assertEqual(plain_content['datas'][2], ('3', '13'))
def test_slice_with_limit_value_of_one(self):
message = self.request_message('SLICE', ['1', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'), ))
def test_batch_with_valid_collection(self):
message = self.request_message('BATCH',
args=[
[(SIGNAL_BATCH_PUT, 'a', 'a'),
(SIGNAL_BATCH_PUT, 'b', 'b'),
(SIGNAL_BATCH_PUT, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_batch_with_invalid_signals(self):
message = self.request_message('BATCH', [
[(-5, 'a', 'a'), (-5, 'b', 'b'), (-5, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)
def test_batch_with_invalid_collection_datas_type(self):
message = self.request_message('BATCH', [
[(SIGNAL_BATCH_PUT, 'a', 1), (SIGNAL_BATCH_PUT, 'b', 2),
(SIGNAL_BATCH_PUT, 'c', 3)],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_connect_to_valid_database(self):
message = Request(
msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['default'],
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsNotNone(plain_content)
def test_connect_to_invalid_database(self):
message = Request(
msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['dadaislikeadad']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_connect_automatically_mounts_and_unmounted_db(self):
# Unmount by hand the database
db_uid = self.handler.databases.index['name_to_uid']['default']
self.handler.databases[
db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
self.handler.databases[db_uid].connector = None
message = Request(
msgpack.packb({
'db_uid': None,
'cmd': 'DBCONNECT',
'args': ['default']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(self.handler.databases[db_uid].status,
self.handler.databases.STATUSES.MOUNTED)
self.assertIsInstance(self.handler.databases[db_uid].connector,
plyvel.DB)
def test_create_valid_db(self):
message = self.request_message('DBCREATE', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_create_already_existing_db(self):
message = self.request_message('DBCREATE', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_drop_valid_db(self):
message = self.request_message('DBDROP', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
# Please teardown with something to destroy
# MOUAHAHAHAH... Hum sorry.
os.mkdir('/tmp/default')
def test_drop_non_existing_db(self):
message = self.request_message('DBDROP', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_list_db(self):
message = self.request_message('DBLIST', [])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], ('default', ))
class Worker(threading.Thread):
STATES = enum('PROCESSING', 'IDLE', 'STOPPED')
def __init__(self, zmq_context, databases, *args, **kwargs):
threading.Thread.__init__(self)
self.instructions = {
WORKER_STATUS: self._status_inst,
WORKER_HALT: self._stop_inst,
WORKER_LAST_ACTION: self._last_activity_inst,
}
self.uid = uuid.uuid4().hex
self.zmq_context = zmq_context
self.state = self.STATES.IDLE
# Wire backend and remote control sockets
self.backend_socket = self.zmq_context.socket(zmq.DEALER)
self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)
self.databases = databases
self.handler = Handler(databases)
self.running = False
self.last_operation = (None, None)
def wire_sockets(self):
"""Connects the worker sockets to their endpoints, and sends
alive signal to the supervisor"""
self.backend_socket.connect('inproc://backend')
self.remote_control_socket.connect('inproc://supervisor')
self.remote_control_socket.send_multipart(
[ServiceMessage.dumps(self.uid)])
def _status_inst(self):
return str(self.state)
def _stop_inst(self):
return self.stop()
def _last_activity_inst(self):
return self.last_operation
def handle_service_message(self):
"""Handles incoming service messages from supervisor socket"""
try:
serialized_request = self.remote_control_socket.recv_multipart(
flags=zmq.NOBLOCK)[0]
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
instruction = ServiceMessage.loads(serialized_request)[0]
try:
response = self.instructions[instruction]()
except KeyError:
errors_logger.exception("%s instruction not recognized by worker" %
instruction)
return
self.remote_control_socket.send_multipart(
[ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)
# If halt instruction succedded, raise HaltException
# so the worker event loop knows it has to stop
if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
raise HaltException()
return
def handle_command(self):
"""Handles incoming command messages from backend socket
Receives incoming messages in a non blocking way,
sets it's set accordingly to IDLE or PROCESSING,
and sends the responses in a non-blocking way.
"""
msg = None
try:
sender_id, msg = self.backend_socket.recv_multipart(
copy=False, flags=zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
self.state = self.STATES.PROCESSING
try:
message = Request(msg)
except MessageFormatError as e:
errors_logger.exception(e.value)
header = ResponseHeader(status=FAILURE_STATUS,
err_code=REQUEST_ERROR,
err_msg=e.value)
content = ResponseContent(datas={})
self.backend_socket.send_multipart([sender_id, header, content],
copy=False)
return
# Handle message, and execute the requested
# command in leveldb
header, response = self.handler.command(message)
self.last_operation = (time.time(), message.db_uid)
self.backend_socket.send_multipart([sender_id, header, response],
flags=zmq.NOBLOCK,
copy=False)
self.state = self.STATES.IDLE
return
def run(self):
"""Non blocking event loop which polls for supervisor
or backend events"""
poller = zmq.Poller()
poller.register(self.backend_socket, zmq.POLLIN)
poller.register(self.remote_control_socket, zmq.POLLIN)
# Connect sockets, and send the supervisor
# alive signals
self.wire_sockets()
while (self.state != self.STATES.STOPPED):
sockets = dict(poller.poll())
if sockets:
if sockets.get(self.remote_control_socket) == zmq.POLLIN:
try:
self.handle_service_message()
except HaltException:
break
if sockets.get(self.backend_socket) == zmq.POLLIN:
self.handle_command() # Might change state
def stop(self):
"""Stops the worker
Changes it's state to STOPPED.
Closes it's backend socket.
Returns SUCCESS_STATUS.
"""
self.state = self.STATES.STOPPED
if not self.backend_socket.closed:
self.backend_socket.close()
activity_logger.info("Gracefully stopping worker %s" % self.uid)
return str(SUCCESS_STATUS)
class ApiTests(unittest2.TestCase):
def _bootstrap_db(self, db):
for val in xrange(9):
db.put(str(val), str(val + 10))
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def tearDown(self):
self.databases.__del__()
del self.handler
os.remove(self.store)
shutil.rmtree(self.dest)
def request_message(self, command, args, db_uid=None):
db_uid = db_uid or self.default_db_uid
return Request(msgpack.packb({
'uid': db_uid,
'cmd': command,
'args': args,
}))
def test_command_with_existing_command(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertNotEqual(plain_content['datas'], None)
def test_command_with_non_existing_command(self):
message = self.request_message('COTCOT', ['testarg'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_command_with_invalid_db_uid(self):
message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)
def test_get_of_existing_key(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11',))
def test_get_of_non_existing_key(self):
message = self.request_message('GET', ['abc123'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_mget_of_existing_keys(self):
message = self.request_message('MGET', [['1', '2', '3']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', '12', '13'))
def test_mget_of_not_fully_existing_keys(self):
message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], WARNING_STATUS)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'], ('11', '12', None))
def test_put_of_valid_key(self):
message = self.request_message('PUT', ['a', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_put_of_invalid_value(self):
message = self.request_message('PUT', ['a', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_delete(self):
message = self.request_message('DELETE', ['9'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_exists_of_existing_key(self):
message = self.request_message('EXISTS', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], True)
def test_exists_of_non_existing_key_1(self):
message = self.request_message('EXISTS', ['0'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_exists_of_non_existing_key_2(self):
message = self.request_message('EXISTS', ['non_existing'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_range(self):
message = self.request_message('RANGE', ['1', '2'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
def test_range_with_keys_only(self):
message = self.request_message('RANGE', ['1', '2', True, False])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas'][0]), 1)
self.assertEqual(len(plain_content['datas'][1]), 1)
self.assertEqual(plain_content['datas'][0], ('1'))
self.assertEqual(plain_content['datas'][1], ('2'))
def test_range_of_len_one(self):
"""Should still return a tuple of tuple"""
message = self.request_message('RANGE', ['1', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'),))
def test_slice_with_limit(self):
message = self.request_message('SLICE', ['1', 3])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
self.assertEqual(plain_content['datas'][2], ('3', '13'))
def test_slice_with_limit_value_of_one(self):
message = self.request_message('SLICE', ['1', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'),))
def test_batch_with_valid_collection(self):
message = self.request_message('BATCH', args=[
[(SIGNAL_BATCH_PUT, 'a', 'a'),
(SIGNAL_BATCH_PUT, 'b', 'b'),
(SIGNAL_BATCH_PUT, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_batch_with_invalid_signals(self):
message = self.request_message('BATCH', [
[(-5, 'a', 'a'),
(-5, 'b', 'b'),
(-5, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)
def test_batch_with_invalid_collection_datas_type(self):
message = self.request_message('BATCH', [
[(SIGNAL_BATCH_PUT, 'a', 1),
(SIGNAL_BATCH_PUT, 'b', 2),
(SIGNAL_BATCH_PUT, 'c', 3)],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_connect_to_valid_database(self):
message = Request(msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['default'],
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsNotNone(plain_content)
def test_connect_to_invalid_database(self):
message = Request(msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['dadaislikeadad']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_connect_automatically_mounts_and_unmounted_db(self):
# Unmount by hand the database
db_uid = self.handler.databases.index['name_to_uid']['default']
self.handler.databases[db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
self.handler.databases[db_uid].connector = None
message = Request(msgpack.packb({
'db_uid': None,
'cmd': 'DBCONNECT',
'args': ['default']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(self.handler.databases[db_uid].status, self.handler.databases.STATUSES.MOUNTED)
self.assertIsInstance(self.handler.databases[db_uid].connector, plyvel.DB)
def test_create_valid_db(self):
message = self.request_message('DBCREATE', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_create_already_existing_db(self):
message = self.request_message('DBCREATE', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_drop_valid_db(self):
message = self.request_message('DBDROP', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
# Please teardown with something to destroy
# MOUAHAHAHAH... Hum sorry.
os.mkdir('/tmp/default')
def test_drop_non_existing_db(self):
message = self.request_message('DBDROP', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_list_db(self):
message = self.request_message('DBLIST', [])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], ('default',))
class Worker(threading.Thread):
STATES = enum("PROCESSING", "IDLE", "STOPPED")
def __init__(self, zmq_context, databases, *args, **kwargs):
threading.Thread.__init__(self)
self.instructions = {
WORKER_STATUS: self._status_inst,
WORKER_HALT: self._stop_inst,
WORKER_LAST_ACTION: self._last_activity_inst,
}
self.uid = uuid.uuid4().hex
self.zmq_context = zmq_context
self.state = self.STATES.IDLE
# Wire backend and remote control sockets
self.backend_socket = self.zmq_context.socket(zmq.DEALER)
self.remote_control_socket = self.zmq_context.socket(zmq.DEALER)
self.databases = databases
self.handler = Handler(databases)
self.running = False
self.last_operation = (None, None)
def wire_sockets(self):
"""Connects the worker sockets to their endpoints, and sends
alive signal to the supervisor"""
self.backend_socket.connect("inproc://backend")
self.remote_control_socket.connect("inproc://supervisor")
self.remote_control_socket.send_multipart([ServiceMessage.dumps(self.uid)])
def _status_inst(self):
return str(self.state)
def _stop_inst(self):
return self.stop()
def _last_activity_inst(self):
return self.last_operation
def handle_service_message(self):
"""Handles incoming service messages from supervisor socket"""
try:
serialized_request = self.remote_control_socket.recv_multipart(flags=zmq.NOBLOCK)[0]
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
instruction = ServiceMessage.loads(serialized_request)[0]
try:
response = self.instructions[instruction]()
except KeyError:
errors_logger.exception("%s instruction not recognized by worker" % instruction)
return
self.remote_control_socket.send_multipart([ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)
# If halt instruction succedded, raise HaltException
# so the worker event loop knows it has to stop
if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
raise HaltException()
return
def handle_command(self):
"""Handles incoming command messages from backend socket
Receives incoming messages in a non blocking way,
sets it's set accordingly to IDLE or PROCESSING,
and sends the responses in a non-blocking way.
"""
msg = None
try:
sender_id, msg = self.backend_socket.recv_multipart(copy=False, flags=zmq.NOBLOCK)
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
self.state = self.STATES.PROCESSING
try:
message = Request(msg)
except MessageFormatError as e:
errors_logger.exception(e.value)
header = ResponseHeader(status=FAILURE_STATUS, err_code=REQUEST_ERROR, err_msg=e.value)
content = ResponseContent(datas={})
self.backend_socket.send_multipart([sender_id, header, content], copy=False)
return
# Handle message, and execute the requested
# command in leveldb
header, response = self.handler.command(message)
self.last_operation = (time.time(), message.db_uid)
self.backend_socket.send_multipart([sender_id, header, response], flags=zmq.NOBLOCK, copy=False)
self.state = self.STATES.IDLE
return
def run(self):
"""Non blocking event loop which polls for supervisor
or backend events"""
poller = zmq.Poller()
poller.register(self.backend_socket, zmq.POLLIN)
poller.register(self.remote_control_socket, zmq.POLLIN)
# Connect sockets, and send the supervisor
# alive signals
self.wire_sockets()
while self.state != self.STATES.STOPPED:
sockets = dict(poller.poll())
if sockets:
if sockets.get(self.remote_control_socket) == zmq.POLLIN:
try:
self.handle_service_message()
except HaltException:
break
if sockets.get(self.backend_socket) == zmq.POLLIN:
self.handle_command() # Might change state
def stop(self):
"""Stops the worker
Changes it's state to STOPPED.
Closes it's backend socket.
Returns SUCCESS_STATUS.
"""
self.state = self.STATES.STOPPED
if not self.backend_socket.closed:
self.backend_socket.close()
activity_logger.info("Gracefully stopping worker %s" % self.uid)
return str(SUCCESS_STATUS)
