# -*- coding: utf-8 -*-
import time
import pyelevator
from pyelevator import WriteBatch, Elevator
# Elevator server holds a default db
# which the client will automatically
# connect to
E = Elevator(transport='tcp', endpoint='192.168.2.11:4141')
ts = te = time.time()
count = 0
'''
batch = E.WriteBatch()
for i in range(0, 1000001):
if count % 10000 == 0:
te = time.time()
batch = E.WriteBatch()
print count, '%s sec' % (te - ts)
ts = te
batch.Put(str(i), str(i))
count += 1
'''
ts = te = time.time()
count = 0
#print E.Slice('1', 10)
for k, v in E.RangeIter('1', '3'):
def setUp(self):
self.client = Elevator(timeout=10)
self._bootstrap_db()
def test_elevator_single_read(n):
for i in xrange(n):
db.Get(str(i))
@timeit
def test_elevator_multi_write(n):
with WriteBatch() as batch:
for i in xrange(n):
batch.Put(str(i), str(i))
if __name__ == '__main__':
n = 100000
db = Elevator()
test_elevator_single_write(n)
test_elevator_single_read(n)
test_elevator_multi_write(n)
"""
# tcp
# elevator -c config/elevator.conf -w 8
'test_elevator_single_write' args: (100000,) 57.43 sec
'test_elevator_single_read' args: (100000,) 64.16 sec
'test_elevator_multi_write' args: (100000,) 0.51 sec
# tcp
# elevator -c config/elevator.conf -w 16
'test_elevator_single_write' args: (100000,) 65.89 sec
'test_elevator_single_read' args: (100000,) 63.12 sec
'test_elevator_multi_write' args: (100000,) 0.50 sec
class BenchElevator(hurdles.BenchCase):
def setUp(self):
self.client = Elevator(timeout=10)
self._bootstrap_db()
def tearDown(self):
pass
def _bootstrap_db(self):
with WriteBatch(timeout=10000) as batch:
for x in xrange(100000):
batch.Put(str(x), uuid.uuid4().hex)
@extra_setup("import uuid\n"
"from pyelevator import WriteBatch\n"
"batch = WriteBatch(timeout=1000)\n"
"for x in xrange(100000):\n"
" batch.Put(str(x), uuid.uuid4().hex)")
def bench_write_batch(self, *args, **kwargs):
kwargs['batch'].Write()
@extra_setup("import random\n"
"keys = [str(random.randint(1, 9999)) for x in xrange(9999)]")
def bench_mget_on_random_keys(self, *args, **kwargs):
self.client.MGet(kwargs['keys'])
@extra_setup("import random\n"
"keys = [str(random.randint(1, 9999)) for x in xrange(9999)]")
def bench_mget_on_random_keys_with_compression(self, *args, **kwargs):
self.client.MGet(kwargs['keys'], compression=True)
@extra_setup("keys = [str(x) for x in xrange(9999)]")
def bench_mget_on_serial_keys(self, *args, **kwargs):
self.client.MGet(kwargs['keys'])
@extra_setup("keys = [str(x) for x in xrange(9999)]")
def bench_mget_on_serial_keys_with_compression(self, *args, **kwargs):
self.client.MGet(kwargs['keys'], compression=True)
def bench_range(self, *args, **kwargs):
self.client.Range('1', '999998')
def bench_range_with_compression(self, *args, **kwargs):
self.client.Range('1', '999998', compression=True)
def bench_slice(self, *args, **kwargs):
self.client.Slice('1', 999998)
def bench_slice_with_compression(self, *args, **kwargs):
self.client.Slice('1', 999998, compression=True)
@extra_setup("import random\n"
"keys = [str(random.randint(1, 10)) for x in xrange(10)]")
def bench_random_get(self, *args, **kwargs):
for key in kwargs['keys']:
self.client.Get(key)
def bench_serial_get(self, *args, **kwargs):
pos = 1
limit = 10
while pos <= limit:
self.client.Get(str(pos))
pos += 1
weibos = db.MGet(weibo_ids)
# weibos = [msgpack.unpackb(weibo) if weibo else None for weibo in weibos]
weibos = [json.loads(weibo) if weibo else None for weibo in weibos]
return weibos
def test_rw(n):
weibos_from_mongo = load_weibos_from_mongo(n)
elevator_multi_write(weibos_from_mongo)
weibo_ids = [str(weibo['id']) for weibo in weibos_from_mongo]
weibos_from_elevator = elevator_multi_read(weibo_ids)
for i in xrange(len(weibos_from_mongo)):
if weibos_from_mongo[i] != weibos_from_elevator[i]:
print '** ' * 10, i
if __name__ == '__main__':
mongo = _default_mongo(usedb='master_timeline')
db = Elevator(timeout=1000)
db.createdb('testdb')
db.connect('testdb')
test_rw(10000)
db.dropdb('testdb')
"""
load 100000 weibos
'load_weibos_from_mongo' args: 7.71 sec
'elevator_multi_read' args: 14.73 sec
结论是elevator并不足以投入prod使用
"""
weibos = db.MGet(weibo_ids)
# weibos = [msgpack.unpackb(weibo) if weibo else None for weibo in weibos]
weibos = [json.loads(weibo) if weibo else None for weibo in weibos]
return weibos
def test_rw(n):
weibos_from_mongo = load_weibos_from_mongo(n)
elevator_multi_write(weibos_from_mongo)
weibo_ids = [str(weibo['id']) for weibo in weibos_from_mongo]
weibos_from_elevator = elevator_multi_read(weibo_ids)
for i in xrange(len(weibos_from_mongo)):
if weibos_from_mongo[i] != weibos_from_elevator[i]:
print '** ' * 10, i
if __name__ == '__main__':
mongo = _default_mongo(usedb='master_timeline')
db = Elevator(timeout=1000)
db.createdb('testdb')
db.connect('testdb')
test_rw(10000)
db.dropdb('testdb')
"""
load 100000 weibos
'load_weibos_from_mongo' args: 7.71 sec
'elevator_multi_read' args: 14.73 sec
结论是elevator并不足以投入prod使用
"""
def setUp(self):
self.elevator_daemon = TestDaemon()
self.elevator_daemon.start()
self.endpoint = '{0}:{1}'.format(self.elevator_daemon.bind, self.elevator_daemon.port)
self.client = Elevator(endpoint=self.endpoint)
self._bootstrap_db()
class ElevatorTest(unittest2.TestCase):
def setUp(self):
self.elevator_daemon = TestDaemon()
self.elevator_daemon.start()
self.endpoint = '{0}:{1}'.format(self.elevator_daemon.bind, self.elevator_daemon.port)
self.client = Elevator(endpoint=self.endpoint)
self._bootstrap_db()
def tearDown(self):
self.elevator_daemon.stop()
del self.elevator_daemon
del self.client
def _bootstrap_db(self):
batch = WriteBatch(endpoint=self.endpoint)
for x in xrange(10):
batch.Put(str(x), str(x + 10))
batch.Write()
def test_connect_store_existing_db(self):
self.client.connect('default')
self.assertIsNotNone(self.client.db_name)
self.assertIsNotNone(self.client.db_uid)
self.assertIsInstance(self.client.db_uid, str)
@raises(DatabaseError)
def test_connect_to_store_non_existing_db(self):
self.client.connect('dadahouse')
def test_createdb_fs_existing_db(self):
tmp_fs_db = os.path.join("/tmp", str(uuid.uuid4()))
db = plyvel.DB(tmp_fs_db, create_if_missing=True)
db.put('abc', '123')
db.put('easy as', 'do re mi')
del db
self.client.createdb(tmp_fs_db)
self.assertIn(tmp_fs_db, self.client.listdb())
self.client.connect(tmp_fs_db)
self.assertEqual(self.client.Get('abc'), '123')
self.assertEqual(self.client.Get('easy as'), 'do re mi')
shutil.rmtree(tmp_fs_db)
def test_createdb_fs_non_db(self):
tmp_fs_db = os.path.join("/tmp", str(uuid.uuid4()))
self.client.createdb(tmp_fs_db)
self.assertIn(tmp_fs_db, self.client.listdb())
self.client.connect(tmp_fs_db)
self.client.Put('abc', '123')
self.client.Put('easy as', 'do re mi')
self.elevator_daemon.stop()
time.sleep(2)
db = plyvel.DB(tmp_fs_db, create_if_missing=True)
self.assertEqual(db.get('abc'), '123')
self.assertEqual(db.get('easy as'), 'do re mi')
shutil.rmtree(tmp_fs_db)
def test_get_with_existing_key(self):
value = self.client.Get('1')
self.assertIsInstance(value, str) # No error (tuple) returned
self.assertEqual(value, '11')
@raises(KeyError)
def test_get_with_invalid_key(self):
self.client.Get('abc')
def test_mget_with_valid_keys(self):
values = self.client.MGet(['1', '2', '3'])
self.assertIsInstance(values, tuple)
self.assertEqual(values, ('11', '12', '13'))
def test_mget_with_invalid_keys(self):
values = self.client.MGet(['1', 'abc', '3'])
self.assertIsInstance(values, tuple)
self.assertEqual(values, ('11', None, '13'))
def test_mget_with_one_existing_key(self):
values = self.client.MGet(['1'])
self.assertIsInstance(values, tuple)
self.assertEqual(values, ('11', ))
def test_mget_with_one_non_existing_key(self):
values = self.client.MGet(['touptoupidou'])
self.assertIsInstance(values, tuple)
self.assertEqual(values, (None, ))
def test_put_valid_value(self):
self.client.Put('abc', '123')
res = self.client.Get('abc')
self.assertIsInstance(res, str)
self.assertEqual(res, '123')
@raises(TypeError)
def test_put_invalid_value(self):
self.client.Put('abc', 123)
def test_delete_existing_key(self):
self.client.Put('abc', '123')
self.client.Delete('abc', '123')
def test_range_of_len_ten(self):
res = self.client.Range('0', '9')
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 10)
for r in res:
self.assertIsNotNone(r)
self.assertIsInstance(r, tuple)
# boostraped values are from 10 to 19
self.assertEqual(int(r[1]), int(r[0]) + 10)
def test_range_of_len_ten_without_keys(self):
res = self.client.Range('0', '9', include_key=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 10)
for r in res:
self.assertIsNotNone(r)
# boostraped values are from 10 to 19
self.assertGreaterEqual(int(r), 10)
self.assertLessEqual(int(r), 19)
def test_range_of_len_ten_without_values(self):
res = self.client.Range('0', '9', include_value=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 10)
for r in res:
self.assertIsNotNone(r)
self.assertGreaterEqual(int(r), 0)
self.assertLessEqual(int(r), 9)
def test_range_of_len_ten_without_keys(self):
res = self.client.Range('0', '9', include_key=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 10)
for r in res:
self.assertIsNotNone(r)
self.assertGreaterEqual(int(r), 0)
self.assertLessEqual(int(r), 9)
def test_range_of_len_ten_without_values(self):
res = self.client.Range('0', '9', include_value=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 10)
for r in res:
self.assertIsNotNone(r)
self.assertGreaterEqual(int(r), 0)
self.assertLessEqual(int(r), 9)
def test_range_of_len_one(self):
res = self.client.Range('1', '1')
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 1)
content = res[0]
self.assertIsInstance(content, tuple)
self.assertEqual(content, ('1', '11'))
def test_range_of_len_one_without_keys(self):
res = self.client.Range('1', '1', include_key=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 1)
self.assertEqual(res, ('11',))
def test_range_of_len_one_without_values(self):
res = self.client.Range('1', '1', include_value=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 1)
self.assertEqual(res, ('1',))
def test_range_of_len_one_without_keys(self):
res = self.client.Range('1', '1', include_key=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 1)
self.assertEqual(res, ('1',))
def test_range_of_len_one_without_values(self):
res = self.client.Range('1', '1', include_value=False)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 1)
self.assertEqual(res, ('1',))
def test_slice_of_len_ten(self):
res = self.client.Slice('0', 9)
self.assertIsInstance(res, tuple)
self.assertEqual(len(res), 9)
for r in res:
self.assertIsNotNone(r)
self.assertIsInstance(r, tuple)
