# -*- coding: utf-8 -*- import time import pyelevator from pyelevator import WriteBatch, Elevator # Elevator server holds a default db # which the client will automatically # connect to E = Elevator(transport='tcp', endpoint='192.168.2.11:4141') ts = te = time.time() count = 0 ''' batch = E.WriteBatch() for i in range(0, 1000001): if count % 10000 == 0: te = time.time() batch = E.WriteBatch() print count, '%s sec' % (te - ts) ts = te batch.Put(str(i), str(i)) count += 1 ''' ts = te = time.time() count = 0 #print E.Slice('1', 10) for k, v in E.RangeIter('1', '3'):
def setUp(self): self.client = Elevator(timeout=10) self._bootstrap_db()
def test_elevator_single_read(n): for i in xrange(n): db.Get(str(i)) @timeit def test_elevator_multi_write(n): with WriteBatch() as batch: for i in xrange(n): batch.Put(str(i), str(i)) if __name__ == '__main__': n = 100000 db = Elevator() test_elevator_single_write(n) test_elevator_single_read(n) test_elevator_multi_write(n) """ # tcp # elevator -c config/elevator.conf -w 8 'test_elevator_single_write' args: (100000,) 57.43 sec 'test_elevator_single_read' args: (100000,) 64.16 sec 'test_elevator_multi_write' args: (100000,) 0.51 sec # tcp # elevator -c config/elevator.conf -w 16 'test_elevator_single_write' args: (100000,) 65.89 sec 'test_elevator_single_read' args: (100000,) 63.12 sec 'test_elevator_multi_write' args: (100000,) 0.50 sec
class BenchElevator(hurdles.BenchCase): def setUp(self): self.client = Elevator(timeout=10) self._bootstrap_db() def tearDown(self): pass def _bootstrap_db(self): with WriteBatch(timeout=10000) as batch: for x in xrange(100000): batch.Put(str(x), uuid.uuid4().hex) @extra_setup("import uuid\n" "from pyelevator import WriteBatch\n" "batch = WriteBatch(timeout=1000)\n" "for x in xrange(100000):\n" " batch.Put(str(x), uuid.uuid4().hex)") def bench_write_batch(self, *args, **kwargs): kwargs['batch'].Write() @extra_setup("import random\n" "keys = [str(random.randint(1, 9999)) for x in xrange(9999)]") def bench_mget_on_random_keys(self, *args, **kwargs): self.client.MGet(kwargs['keys']) @extra_setup("import random\n" "keys = [str(random.randint(1, 9999)) for x in xrange(9999)]") def bench_mget_on_random_keys_with_compression(self, *args, **kwargs): self.client.MGet(kwargs['keys'], compression=True) @extra_setup("keys = [str(x) for x in xrange(9999)]") def bench_mget_on_serial_keys(self, *args, **kwargs): self.client.MGet(kwargs['keys']) @extra_setup("keys = [str(x) for x in xrange(9999)]") def bench_mget_on_serial_keys_with_compression(self, *args, **kwargs): self.client.MGet(kwargs['keys'], compression=True) def bench_range(self, *args, **kwargs): self.client.Range('1', '999998') def bench_range_with_compression(self, *args, **kwargs): self.client.Range('1', '999998', compression=True) def bench_slice(self, *args, **kwargs): self.client.Slice('1', 999998) def bench_slice_with_compression(self, *args, **kwargs): self.client.Slice('1', 999998, compression=True) @extra_setup("import random\n" "keys = [str(random.randint(1, 10)) for x in xrange(10)]") def bench_random_get(self, *args, **kwargs): for key in kwargs['keys']: self.client.Get(key) def bench_serial_get(self, *args, **kwargs): pos = 1 limit = 10 while pos <= limit: self.client.Get(str(pos)) pos += 1
weibos = db.MGet(weibo_ids) # weibos = [msgpack.unpackb(weibo) if weibo else None for weibo in weibos] weibos = [json.loads(weibo) if weibo else None for weibo in weibos] return weibos def test_rw(n): weibos_from_mongo = load_weibos_from_mongo(n) elevator_multi_write(weibos_from_mongo) weibo_ids = [str(weibo['id']) for weibo in weibos_from_mongo] weibos_from_elevator = elevator_multi_read(weibo_ids) for i in xrange(len(weibos_from_mongo)): if weibos_from_mongo[i] != weibos_from_elevator[i]: print '** ' * 10, i if __name__ == '__main__': mongo = _default_mongo(usedb='master_timeline') db = Elevator(timeout=1000) db.createdb('testdb') db.connect('testdb') test_rw(10000) db.dropdb('testdb') """ load 100000 weibos 'load_weibos_from_mongo' args: 7.71 sec 'elevator_multi_read' args: 14.73 sec 结论是elevator并不足以投入prod使用 """
def setUp(self): self.elevator_daemon = TestDaemon() self.elevator_daemon.start() self.endpoint = '{0}:{1}'.format(self.elevator_daemon.bind, self.elevator_daemon.port) self.client = Elevator(endpoint=self.endpoint) self._bootstrap_db()
class ElevatorTest(unittest2.TestCase): def setUp(self): self.elevator_daemon = TestDaemon() self.elevator_daemon.start() self.endpoint = '{0}:{1}'.format(self.elevator_daemon.bind, self.elevator_daemon.port) self.client = Elevator(endpoint=self.endpoint) self._bootstrap_db() def tearDown(self): self.elevator_daemon.stop() del self.elevator_daemon del self.client def _bootstrap_db(self): batch = WriteBatch(endpoint=self.endpoint) for x in xrange(10): batch.Put(str(x), str(x + 10)) batch.Write() def test_connect_store_existing_db(self): self.client.connect('default') self.assertIsNotNone(self.client.db_name) self.assertIsNotNone(self.client.db_uid) self.assertIsInstance(self.client.db_uid, str) @raises(DatabaseError) def test_connect_to_store_non_existing_db(self): self.client.connect('dadahouse') def test_createdb_fs_existing_db(self): tmp_fs_db = os.path.join("/tmp", str(uuid.uuid4())) db = plyvel.DB(tmp_fs_db, create_if_missing=True) db.put('abc', '123') db.put('easy as', 'do re mi') del db self.client.createdb(tmp_fs_db) self.assertIn(tmp_fs_db, self.client.listdb()) self.client.connect(tmp_fs_db) self.assertEqual(self.client.Get('abc'), '123') self.assertEqual(self.client.Get('easy as'), 'do re mi') shutil.rmtree(tmp_fs_db) def test_createdb_fs_non_db(self): tmp_fs_db = os.path.join("/tmp", str(uuid.uuid4())) self.client.createdb(tmp_fs_db) self.assertIn(tmp_fs_db, self.client.listdb()) self.client.connect(tmp_fs_db) self.client.Put('abc', '123') self.client.Put('easy as', 'do re mi') self.elevator_daemon.stop() time.sleep(2) db = plyvel.DB(tmp_fs_db, create_if_missing=True) self.assertEqual(db.get('abc'), '123') self.assertEqual(db.get('easy as'), 'do re mi') shutil.rmtree(tmp_fs_db) def test_get_with_existing_key(self): value = self.client.Get('1') self.assertIsInstance(value, str) # No error (tuple) returned self.assertEqual(value, '11') @raises(KeyError) def test_get_with_invalid_key(self): self.client.Get('abc') def test_mget_with_valid_keys(self): values = self.client.MGet(['1', '2', '3']) self.assertIsInstance(values, tuple) self.assertEqual(values, ('11', '12', '13')) def test_mget_with_invalid_keys(self): values = self.client.MGet(['1', 'abc', '3']) self.assertIsInstance(values, tuple) self.assertEqual(values, ('11', None, '13')) def test_mget_with_one_existing_key(self): values = self.client.MGet(['1']) self.assertIsInstance(values, tuple) self.assertEqual(values, ('11', )) def test_mget_with_one_non_existing_key(self): values = self.client.MGet(['touptoupidou']) self.assertIsInstance(values, tuple) self.assertEqual(values, (None, )) def test_put_valid_value(self): self.client.Put('abc', '123') res = self.client.Get('abc') self.assertIsInstance(res, str) self.assertEqual(res, '123') @raises(TypeError) def test_put_invalid_value(self): self.client.Put('abc', 123) def test_delete_existing_key(self): self.client.Put('abc', '123') self.client.Delete('abc', '123') def test_range_of_len_ten(self): res = self.client.Range('0', '9') self.assertIsInstance(res, tuple) self.assertEqual(len(res), 10) for r in res: self.assertIsNotNone(r) self.assertIsInstance(r, tuple) # boostraped values are from 10 to 19 self.assertEqual(int(r[1]), int(r[0]) + 10) def test_range_of_len_ten_without_keys(self): res = self.client.Range('0', '9', include_key=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 10) for r in res: self.assertIsNotNone(r) # boostraped values are from 10 to 19 self.assertGreaterEqual(int(r), 10) self.assertLessEqual(int(r), 19) def test_range_of_len_ten_without_values(self): res = self.client.Range('0', '9', include_value=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 10) for r in res: self.assertIsNotNone(r) self.assertGreaterEqual(int(r), 0) self.assertLessEqual(int(r), 9) def test_range_of_len_ten_without_keys(self): res = self.client.Range('0', '9', include_key=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 10) for r in res: self.assertIsNotNone(r) self.assertGreaterEqual(int(r), 0) self.assertLessEqual(int(r), 9) def test_range_of_len_ten_without_values(self): res = self.client.Range('0', '9', include_value=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 10) for r in res: self.assertIsNotNone(r) self.assertGreaterEqual(int(r), 0) self.assertLessEqual(int(r), 9) def test_range_of_len_one(self): res = self.client.Range('1', '1') self.assertIsInstance(res, tuple) self.assertEqual(len(res), 1) content = res[0] self.assertIsInstance(content, tuple) self.assertEqual(content, ('1', '11')) def test_range_of_len_one_without_keys(self): res = self.client.Range('1', '1', include_key=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 1) self.assertEqual(res, ('11',)) def test_range_of_len_one_without_values(self): res = self.client.Range('1', '1', include_value=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 1) self.assertEqual(res, ('1',)) def test_range_of_len_one_without_keys(self): res = self.client.Range('1', '1', include_key=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 1) self.assertEqual(res, ('1',)) def test_range_of_len_one_without_values(self): res = self.client.Range('1', '1', include_value=False) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 1) self.assertEqual(res, ('1',)) def test_slice_of_len_ten(self): res = self.client.Slice('0', 9) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 9) for r in res: self.assertIsNotNone(r) self.assertIsInstance(r, tuple)