def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.handler = DatabaseStore(self.config)
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
class MajordomeTest(unittest2.TestCase):
def setUp(self):
zmq_context = zmq.Context()
env = gen_test_env()
self.database_store = env['global']['database_store']
self.databases_storage_path = env['global']['databases_storage_path']
if not os.path.exists(self.databases_storage_path):
os.mkdir(self.databases_storage_path)
self.db_handler = DatabaseStore(self.database_store,
self.databases_storage_path)
# Let's fake a backend for workers to talk to
self.socket = zmq_context.socket(zmq.DEALER)
self.socket.bind('inproc://backend')
self.supervisor = Supervisor(zmq_context, self.db_handler)
def tearDown(self):
self.supervisor.stop_all()
if hasattr(self, 'majordome'):
self.majordome.cancel()
os.remove(self.database_store)
shutil.rmtree(self.databases_storage_path)
def test_unmount_existing_mounted_database(self):
default_db_uid = self.db_handler.index['name_to_uid']['default']
db_to_watch = self.db_handler[default_db_uid]
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
# Use 1/60 interval in order to set it as 1sec
self.majordome = Majordome(self.supervisor,
self.db_handler,
1 / 60)
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
def test_unmount_existing_unmounted_database(self):
default_db_uid = self.db_handler.index['name_to_uid']['default']
db_to_watch = self.db_handler[default_db_uid]
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
self.db_handler.umount('default')
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.UNMOUNTED)
# Use 1/60 interval in order to set it as 1sec
self.majordome = Majordome(self.supervisor,
self.db_handler,
1 / 60)
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.UNMOUNTED)
class MajordomeTest(unittest2.TestCase):
def setUp(self):
zmq_context = zmq.Context()
config = gen_test_config()
self.database_store = config['database_store']
self.databases_storage_path = config['databases_storage_path']
if not os.path.exists(self.databases_storage_path):
os.mkdir(self.databases_storage_path)
self.db_handler = DatabaseStore(config)
# Let's fake a backend for workers to talk to
self.socket = zmq_context.socket(zmq.DEALER)
self.socket.bind('inproc://backend')
self.supervisor = Supervisor(zmq_context, self.db_handler)
def tearDown(self):
self.supervisor.stop_all()
if hasattr(self, 'majordome'):
self.majordome.cancel()
os.remove(self.database_store)
shutil.rmtree(self.databases_storage_path)
def test_unmount_existing_mounted_database(self):
default_db_uid = self.db_handler.index['name_to_uid']['default']
db_to_watch = self.db_handler[default_db_uid]
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
# Use 1/60 interval in order to set it as 1sec
self.majordome = Majordome(self.supervisor, self.db_handler, 1 / 60)
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
def test_unmount_existing_unmounted_database(self):
default_db_uid = self.db_handler.index['name_to_uid']['default']
db_to_watch = self.db_handler[default_db_uid]
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.MOUNTED)
self.db_handler.umount('default')
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.UNMOUNTED)
# Use 1/60 interval in order to set it as 1sec
self.majordome = Majordome(self.supervisor, self.db_handler, 1 / 60)
self.assertEqual(db_to_watch.status, DatabaseStore.STATUSES.UNMOUNTED)
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.handler = DatabaseStore(self.config)
def setUp(self):
zmq_context = zmq.Context()
config = gen_test_config()
self.database_store = config['database_store']
self.databases_storage_path = config['databases_storage_path']
if not os.path.exists(self.databases_storage_path):
os.mkdir(self.databases_storage_path)
self.db_handler = DatabaseStore(config)
# Let's fake a backend for workers to talk to
self.socket = zmq_context.socket(zmq.DEALER)
self.socket.bind('inproc://backend')
self.supervisor = Supervisor(zmq_context, self.db_handler)
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def setUp(self):
zmq_context = zmq.Context()
config = gen_test_config()
self.database_store = config['database_store']
self.databases_storage_path = config['databases_storage_path']
if not os.path.exists(self.databases_storage_path):
os.mkdir(self.databases_storage_path)
self.db_handler = DatabaseStore(config)
# Let's fake a backend for workers to talk to
self.socket = zmq_context.socket(zmq.DEALER)
self.socket.bind('inproc://backend')
self.supervisor = Supervisor(zmq_context, self.db_handler)
def runserver(config):
setup_loggers(config)
activity_logger = logging.getLogger("activity_logger")
databases = DatabaseStore(config)
backend = Backend(databases, config)
frontend = Frontend(config)
poller = zmq.Poller()
poller.register(backend.socket, zmq.POLLIN)
poller.register(frontend.socket, zmq.POLLIN)
activity_logger.info('Elevator server started on %s' % frontend.host)
while True:
try:
sockets = dict(poller.poll())
if frontend.socket in sockets:
if sockets[frontend.socket] == zmq.POLLIN:
msg = frontend.socket.recv_multipart(copy=False)
backend.socket.send_multipart(msg, copy=False)
if backend.socket in sockets:
if sockets[backend.socket] == zmq.POLLIN:
msg = backend.socket.recv_multipart(copy=False)
frontend.socket.send_multipart(msg, copy=False)
except KeyboardInterrupt:
activity_logger.info('Gracefully shuthing down workers')
del backend
activity_logger.info('Stopping frontend')
del frontend
activity_logger.info('Done')
return
except Exception as e:
log_critical(e)
del backend
del frontend
return
class ApiTests(unittest2.TestCase):
def _bootstrap_db(self, db):
for val in xrange(9):
db.put(str(val), str(val + 10))
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def tearDown(self):
self.databases.__del__()
del self.handler
os.remove(self.store)
shutil.rmtree(self.dest)
def request_message(self, command, args, db_uid=None):
db_uid = db_uid or self.default_db_uid
return Request(
msgpack.packb({
'uid': db_uid,
'cmd': command,
'args': args,
}))
def test_command_with_existing_command(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertNotEqual(plain_content['datas'], None)
def test_command_with_non_existing_command(self):
message = self.request_message('COTCOT', ['testarg'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_command_with_invalid_db_uid(self):
message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)
def test_get_of_existing_key(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', ))
def test_get_of_non_existing_key(self):
message = self.request_message('GET', ['abc123'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_mget_of_existing_keys(self):
message = self.request_message('MGET', [['1', '2', '3']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', '12', '13'))
def test_mget_of_not_fully_existing_keys(self):
message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], WARNING_STATUS)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'], ('11', '12', None))
def test_put_of_valid_key(self):
message = self.request_message('PUT', ['a', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_put_of_invalid_value(self):
message = self.request_message('PUT', ['a', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_delete(self):
message = self.request_message('DELETE', ['9'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_exists_of_existing_key(self):
message = self.request_message('EXISTS', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], True)
def test_exists_of_non_existing_key_1(self):
message = self.request_message('EXISTS', ['0'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_exists_of_non_existing_key_2(self):
message = self.request_message('EXISTS', ['non_existing'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_range(self):
message = self.request_message('RANGE', ['1', '2'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
def test_range_with_keys_only(self):
message = self.request_message('RANGE', ['1', '2', True, False])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas'][0]), 1)
self.assertEqual(len(plain_content['datas'][1]), 1)
self.assertEqual(plain_content['datas'][0], ('1'))
self.assertEqual(plain_content['datas'][1], ('2'))
def test_range_of_len_one(self):
"""Should still return a tuple of tuple"""
message = self.request_message('RANGE', ['1', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'), ))
def test_slice_with_limit(self):
message = self.request_message('SLICE', ['1', 3])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
self.assertEqual(plain_content['datas'][2], ('3', '13'))
def test_slice_with_limit_value_of_one(self):
message = self.request_message('SLICE', ['1', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'), ))
def test_batch_with_valid_collection(self):
message = self.request_message('BATCH',
args=[
[(SIGNAL_BATCH_PUT, 'a', 'a'),
(SIGNAL_BATCH_PUT, 'b', 'b'),
(SIGNAL_BATCH_PUT, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_batch_with_invalid_signals(self):
message = self.request_message('BATCH', [
[(-5, 'a', 'a'), (-5, 'b', 'b'), (-5, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)
def test_batch_with_invalid_collection_datas_type(self):
message = self.request_message('BATCH', [
[(SIGNAL_BATCH_PUT, 'a', 1), (SIGNAL_BATCH_PUT, 'b', 2),
(SIGNAL_BATCH_PUT, 'c', 3)],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_connect_to_valid_database(self):
message = Request(
msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['default'],
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsNotNone(plain_content)
def test_connect_to_invalid_database(self):
message = Request(
msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['dadaislikeadad']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_connect_automatically_mounts_and_unmounted_db(self):
# Unmount by hand the database
db_uid = self.handler.databases.index['name_to_uid']['default']
self.handler.databases[
db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
self.handler.databases[db_uid].connector = None
message = Request(
msgpack.packb({
'db_uid': None,
'cmd': 'DBCONNECT',
'args': ['default']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(self.handler.databases[db_uid].status,
self.handler.databases.STATUSES.MOUNTED)
self.assertIsInstance(self.handler.databases[db_uid].connector,
plyvel.DB)
def test_create_valid_db(self):
message = self.request_message('DBCREATE', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_create_already_existing_db(self):
message = self.request_message('DBCREATE', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_drop_valid_db(self):
message = self.request_message('DBDROP', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
# Please teardown with something to destroy
# MOUAHAHAHAH... Hum sorry.
os.mkdir('/tmp/default')
def test_drop_non_existing_db(self):
message = self.request_message('DBDROP', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_list_db(self):
message = self.request_message('DBLIST', [])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], ('default', ))
class DatabasesTest(unittest2.TestCase):
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.handler = DatabaseStore(self.config)
def tearDown(self):
self.handler.__del__()
os.remove('/tmp/store.json')
shutil.rmtree('/tmp/dbs')
def test_init(self):
self.assertIn('default', self.handler.index['name_to_uid'])
default_db_uid = self.handler.index['name_to_uid']['default']
self.assertEqual(self.handler[default_db_uid].name, 'default')
self.assertEqual(self.handler[default_db_uid].path, '/tmp/dbs/default')
def test_load(self):
db_name = 'testdb'
self.handler.add(db_name)
self.assertIn(db_name, self.handler.index['name_to_uid'])
db_uid = self.handler.index['name_to_uid'][db_name]
self.assertIn(db_uid, self.handler)
self.assertEqual(self.handler[db_uid].name, db_name)
self.assertIn('default', self.handler.index['name_to_uid'])
def test_store_update(self):
db_name = 'test_db'
db_desc = {
'path': '/tmp/test_path',
'uid': 'testuid',
'options': {},
}
self.handler.store_update(db_name, db_desc)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name], db_desc)
def test_store_remove(self):
db_name = 'test_db'
db_desc = {
'path': '/tmp/test_path',
'uid': 'testuid',
'options': {},
}
self.handler.store_update(db_name, db_desc)
self.handler.store_remove(db_name)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_drop_existing_db(self):
db_name = 'default' # Automatically created on startup
status, content = self.handler.drop(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_remove_existing_db_which_files_were_erased(self):
db_name = 'testdb' # Automatically created on startup
db_path = '/tmp/dbs/testdb'
status, content = self.handler.add(db_name)
shutil.rmtree(db_path)
status, content = self.handler.drop(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertIsInstance(content, list)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_add_from_db_name_without_options_passed(self):
db_name = 'testdb'
default_db_options = DatabaseOptions()
status, content = self.handler.add(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name]["path"],
os.path.join(self.dest, db_name))
self.assertIsNotNone(store_datas[db_name]["uid"])
stored_db_options = store_datas[db_name]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in default_db_options.iteritems():
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_name_with_options_passed(self):
db_name = 'testdb'
db_options = DatabaseOptions(paranoid_checks=True)
status, content = self.handler.add(db_name, db_options)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name]["path"],
os.path.join(self.dest, db_name))
self.assertIsNotNone(store_datas[db_name]["uid"])
stored_db_options = store_datas[db_name]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in db_options.iteritems():
if option_name == "paranoid_check":
self.assertEqual(option_value, False)
continue
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_abspath(self):
db_path = '/tmp/dbs/testdb' # Could be anywhere on fs
default_db_options = DatabaseOptions()
status, content = self.handler.add(db_path)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_path, store_datas)
self.assertEqual(store_datas[db_path]["path"],
os.path.join(self.dest, db_path))
self.assertIsNotNone(store_datas[db_path]["uid"])
stored_db_options = store_datas[db_path]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in default_db_options.iteritems():
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_relpath(self):
db_path = './testdb' # Could be anywhere on fs
status, content = self.handler.add(db_path)
self.assertEqual(status, FAILURE_STATUS)
self.assertIsInstance(content, list)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_path, store_datas)
def test_add_db_mounts_it_automatically(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
self.assertEqual(self.handler[db_uid].status, Database.STATUS.MOUNTED)
self.assertIsNotNone(self.handler[db_uid].connector)
self.assertIsInstance(self.handler[db_uid].connector, plyvel.DB)
def test_mount_unmounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Unmount the db by hand
self.handler[db_uid].status = self.handler.STATUSES.UNMOUNTED
self.handler[db_uid].connector = None
# Re-mount it and assert everything went fine
status, content = self.handler.mount(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(self.handler[db_uid].status, self.handler.STATUSES.MOUNTED)
self.assertIsNotNone(self.handler[db_uid].connector)
self.assertIsInstance(self.handler[db_uid].connector, plyvel.DB)
def test_mount_already_mounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
status, content = self.handler.mount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
def test_mount_corrupted_db(self):
db_name = 'testdb'
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Intetionaly rm db MANIFEST in order for a corruption
# to appear.
rm_from_pattern(self.handler[db_uid].path, 'MANIFEST*')
status, content = self.handler.mount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
def test_unmount_mounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Re-mount it and assert everything went fine
status, content = self.handler.umount(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(self.handler[db_uid].status, self.handler.STATUSES.UNMOUNTED)
self.assertIsNone(self.handler[db_uid].connector)
def test_umount_already_unmounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Unmount the db by hand
self.handler[db_uid].status = self.handler.STATUSES.UNMOUNTED
self.handler[db_uid].connector = None
status, content = self.handler.umount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
class ApiTests(unittest2.TestCase):
def _bootstrap_db(self, db):
for val in xrange(9):
db.put(str(val), str(val + 10))
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.databases = DatabaseStore(self.config)
self.default_db_uid = self.databases.index['name_to_uid']['default']
self._bootstrap_db(self.databases[self.default_db_uid].connector)
self.handler = Handler(self.databases)
def tearDown(self):
self.databases.__del__()
del self.handler
os.remove(self.store)
shutil.rmtree(self.dest)
def request_message(self, command, args, db_uid=None):
db_uid = db_uid or self.default_db_uid
return Request(msgpack.packb({
'uid': db_uid,
'cmd': command,
'args': args,
}))
def test_command_with_existing_command(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertNotEqual(plain_content['datas'], None)
def test_command_with_non_existing_command(self):
message = self.request_message('COTCOT', ['testarg'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_command_with_invalid_db_uid(self):
message = self.request_message('PUT', ['1', '1'], db_uid='failinguid')
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], RUNTIME_ERROR)
def test_get_of_existing_key(self):
message = self.request_message('GET', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11',))
def test_get_of_non_existing_key(self):
message = self.request_message('GET', ['abc123'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], KEY_ERROR)
def test_mget_of_existing_keys(self):
message = self.request_message('MGET', [['1', '2', '3']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], ('11', '12', '13'))
def test_mget_of_not_fully_existing_keys(self):
message = self.request_message('MGET', [['1', '2', 'touptoupidou']])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], WARNING_STATUS)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'], ('11', '12', None))
def test_put_of_valid_key(self):
message = self.request_message('PUT', ['a', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_put_of_invalid_value(self):
message = self.request_message('PUT', ['a', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_delete(self):
message = self.request_message('DELETE', ['9'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_exists_of_existing_key(self):
message = self.request_message('EXISTS', ['1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], True)
def test_exists_of_non_existing_key_1(self):
message = self.request_message('EXISTS', ['0'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_exists_of_non_existing_key_2(self):
message = self.request_message('EXISTS', ['non_existing'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], False)
def test_range(self):
message = self.request_message('RANGE', ['1', '2'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
def test_range_with_keys_only(self):
message = self.request_message('RANGE', ['1', '2', True, False])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas'][0]), 1)
self.assertEqual(len(plain_content['datas'][1]), 1)
self.assertEqual(plain_content['datas'][0], ('1'))
self.assertEqual(plain_content['datas'][1], ('2'))
def test_range_of_len_one(self):
"""Should still return a tuple of tuple"""
message = self.request_message('RANGE', ['1', '1'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'),))
def test_slice_with_limit(self):
message = self.request_message('SLICE', ['1', 3])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content['datas']), 3)
self.assertEqual(plain_content['datas'][0], ('1', '11'))
self.assertEqual(plain_content['datas'][1], ('2', '12'))
self.assertEqual(plain_content['datas'][2], ('3', '13'))
def test_slice_with_limit_value_of_one(self):
message = self.request_message('SLICE', ['1', 1])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsInstance(plain_content['datas'], tuple)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], (('1', '11'),))
def test_batch_with_valid_collection(self):
message = self.request_message('BATCH', args=[
[(SIGNAL_BATCH_PUT, 'a', 'a'),
(SIGNAL_BATCH_PUT, 'b', 'b'),
(SIGNAL_BATCH_PUT, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_batch_with_invalid_signals(self):
message = self.request_message('BATCH', [
[(-5, 'a', 'a'),
(-5, 'b', 'b'),
(-5, 'c', 'c')],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], SIGNAL_ERROR)
def test_batch_with_invalid_collection_datas_type(self):
message = self.request_message('BATCH', [
[(SIGNAL_BATCH_PUT, 'a', 1),
(SIGNAL_BATCH_PUT, 'b', 2),
(SIGNAL_BATCH_PUT, 'c', 3)],
])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], TYPE_ERROR)
def test_connect_to_valid_database(self):
message = Request(msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['default'],
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertIsNotNone(plain_content)
def test_connect_to_invalid_database(self):
message = Request(msgpack.packb({
'uid': None,
'cmd': 'DBCONNECT',
'args': ['dadaislikeadad']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_connect_automatically_mounts_and_unmounted_db(self):
# Unmount by hand the database
db_uid = self.handler.databases.index['name_to_uid']['default']
self.handler.databases[db_uid].status = self.handler.databases.STATUSES.UNMOUNTED
self.handler.databases[db_uid].connector = None
message = Request(msgpack.packb({
'db_uid': None,
'cmd': 'DBCONNECT',
'args': ['default']
}))
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(self.handler.databases[db_uid].status, self.handler.databases.STATUSES.MOUNTED)
self.assertIsInstance(self.handler.databases[db_uid].connector, plyvel.DB)
def test_create_valid_db(self):
message = self.request_message('DBCREATE', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
def test_create_already_existing_db(self):
message = self.request_message('DBCREATE', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_drop_valid_db(self):
message = self.request_message('DBDROP', ['default'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(plain_content['datas'], None)
# Please teardown with something to destroy
# MOUAHAHAHAH... Hum sorry.
os.mkdir('/tmp/default')
def test_drop_non_existing_db(self):
message = self.request_message('DBDROP', ['testdb'])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
self.assertEqual(plain_header['status'], FAILURE_STATUS)
self.assertEqual(plain_header['err_code'], DATABASE_ERROR)
def test_list_db(self):
message = self.request_message('DBLIST', [])
header, content = self.handler.command(message)
plain_header = msgpack.unpackb(header)
plain_content = msgpack.unpackb(content)
self.assertEqual(plain_header['status'], SUCCESS_STATUS)
self.assertEqual(len(plain_content), 1)
self.assertEqual(plain_content['datas'], ('default',))
class DatabasesTest(unittest2.TestCase):
def setUp(self):
self.store = '/tmp/store.json'
self.dest = '/tmp/dbs'
self.config = gen_test_config()
if not os.path.exists(self.dest):
os.mkdir(self.dest)
self.handler = DatabaseStore(self.config)
def tearDown(self):
self.handler.__del__()
os.remove('/tmp/store.json')
shutil.rmtree('/tmp/dbs')
def test_init(self):
self.assertIn('default', self.handler.index['name_to_uid'])
default_db_uid = self.handler.index['name_to_uid']['default']
self.assertEqual(self.handler[default_db_uid].name, 'default')
self.assertEqual(self.handler[default_db_uid].path, '/tmp/dbs/default')
def test_load(self):
db_name = 'testdb'
self.handler.add(db_name)
self.assertIn(db_name, self.handler.index['name_to_uid'])
db_uid = self.handler.index['name_to_uid'][db_name]
self.assertIn(db_uid, self.handler)
self.assertEqual(self.handler[db_uid].name, db_name)
self.assertIn('default', self.handler.index['name_to_uid'])
def test_store_update(self):
db_name = 'test_db'
db_desc = {
'path': '/tmp/test_path',
'uid': 'testuid',
'options': {},
}
self.handler.store_update(db_name, db_desc)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name], db_desc)
def test_store_remove(self):
db_name = 'test_db'
db_desc = {
'path': '/tmp/test_path',
'uid': 'testuid',
'options': {},
}
self.handler.store_update(db_name, db_desc)
self.handler.store_remove(db_name)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_drop_existing_db(self):
db_name = 'default' # Automatically created on startup
status, content = self.handler.drop(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_remove_existing_db_which_files_were_erased(self):
db_name = 'testdb' # Automatically created on startup
db_path = '/tmp/dbs/testdb'
status, content = self.handler.add(db_name)
shutil.rmtree(db_path)
status, content = self.handler.drop(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertIsInstance(content, list)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_name, store_datas)
def test_add_from_db_name_without_options_passed(self):
db_name = 'testdb'
default_db_options = DatabaseOptions()
status, content = self.handler.add(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name]["path"],
os.path.join(self.dest, db_name))
self.assertIsNotNone(store_datas[db_name]["uid"])
stored_db_options = store_datas[db_name]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in default_db_options.iteritems():
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_name_with_options_passed(self):
db_name = 'testdb'
db_options = DatabaseOptions(paranoid_checks=True)
status, content = self.handler.add(db_name, db_options)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_name, store_datas)
self.assertEqual(store_datas[db_name]["path"],
os.path.join(self.dest, db_name))
self.assertIsNotNone(store_datas[db_name]["uid"])
stored_db_options = store_datas[db_name]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in db_options.iteritems():
if option_name == "paranoid_check":
self.assertEqual(option_value, False)
continue
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_abspath(self):
db_path = '/tmp/dbs/testdb' # Could be anywhere on fs
default_db_options = DatabaseOptions()
status, content = self.handler.add(db_path)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(content, None)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertIn(db_path, store_datas)
self.assertEqual(store_datas[db_path]["path"],
os.path.join(self.dest, db_path))
self.assertIsNotNone(store_datas[db_path]["uid"])
stored_db_options = store_datas[db_path]["options"]
self.assertIsNotNone(stored_db_options)
self.assertIsInstance(stored_db_options, dict)
for option_name, option_value in default_db_options.iteritems():
self.assertIn(option_name, stored_db_options)
self.assertEqual(option_value, stored_db_options[option_name])
def test_add_from_db_relpath(self):
db_path = './testdb' # Could be anywhere on fs
status, content = self.handler.add(db_path)
self.assertEqual(status, FAILURE_STATUS)
self.assertIsInstance(content, list)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
store_datas = json.load(open(self.handler.store_file, 'r'))
self.assertNotIn(db_path, store_datas)
def test_add_db_mounts_it_automatically(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
self.assertEqual(self.handler[db_uid].status, Database.STATUS.MOUNTED)
self.assertIsNotNone(self.handler[db_uid].connector)
self.assertIsInstance(self.handler[db_uid].connector, plyvel.DB)
def test_mount_unmounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Unmount the db by hand
self.handler[db_uid].status = self.handler.STATUSES.UNMOUNTED
self.handler[db_uid].connector = None
# Re-mount it and assert everything went fine
status, content = self.handler.mount(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(self.handler[db_uid].status,
self.handler.STATUSES.MOUNTED)
self.assertIsNotNone(self.handler[db_uid].connector)
self.assertIsInstance(self.handler[db_uid].connector, plyvel.DB)
def test_mount_already_mounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
status, content = self.handler.mount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
def test_mount_corrupted_db(self):
db_name = 'testdb'
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Intetionaly rm db MANIFEST in order for a corruption
# to appear.
rm_from_pattern(self.handler[db_uid].path, 'MANIFEST*')
status, content = self.handler.mount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
def test_unmount_mounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Re-mount it and assert everything went fine
status, content = self.handler.umount(db_name)
self.assertEqual(status, SUCCESS_STATUS)
self.assertEqual(self.handler[db_uid].status,
self.handler.STATUSES.UNMOUNTED)
self.assertIsNone(self.handler[db_uid].connector)
def test_umount_already_unmounted_db(self):
db_name = 'testdb' # Automatically created on startup
status, content = self.handler.add(db_name)
db_uid = self.handler.index['name_to_uid'][db_name]
# Unmount the db by hand
self.handler[db_uid].status = self.handler.STATUSES.UNMOUNTED
self.handler[db_uid].connector = None
status, content = self.handler.umount(db_name)
self.assertEqual(status, FAILURE_STATUS)
self.assertEqual(len(content), 2)
self.assertEqual(content[0], DATABASE_ERROR)
