class TestBrokenBase(TestGCBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
#
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=1)
# only generate keys in sector0
def _gen_data(self, data, prefix='', loop_num=10 * 1024, sector=0):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix,
count=loop_num,
sector=sector):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
store.close()
def _check_data(self, data, prefix='', loop_num=10 * 1024, sector=0):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix,
count=loop_num,
sector=sector):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
store.close()
class TestBrokenBase(TestGCBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
#
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=1)
# only generate keys in sector0
def _gen_data(self, data, prefix='', loop_num=10 * 1024, sector=0):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
store.close()
def _check_data(self, data, prefix='', loop_num=10 * 1024, sector=0):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
store.close()
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.last_pos = 0
self.last_size = 0
class TestGCMultiple2(TestGCBase):
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, accesslog=False, max_data_size=10, db_depth=2)
# buffer size is 4m, max_data_size set to 10m for data file better reach above 6m
# turn off accesslog to speed up write
# only generate keys in sector0
def _gen_data(self, data, prefix="", loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def _delete_data(self, prefix="", loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def _check_data(self, data, prefix="", loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
class TestGCMultiple2(TestGCBase):
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, accesslog=False, max_data_size=10, db_depth=2)
# buffer size is 4m, max_data_size set to 10m for data file better reach above 6m
# turn off accesslog to speed up write
# only generate keys in sector0
def _gen_data(self, data, prefix='', loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def _delete_data(self, prefix='', loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def _check_data(self, data, prefix='', loop_num=10 * 1024, sector=(0, 0)):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
class TestGCBase(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
def _start_gc(self, ignore_recent=0, bucket=None):
""" bucket must be in 0 or 00 string """
if bucket is not None:
assert isinstance(bucket, basestring) and len(bucket) <= 2
t = telnetlib.Telnet("127.0.0.1", self.backend1.port)
if bucket is None:
t.write('flush_all {}\n'.format(ignore_recent))
else:
t.write('flush_all %s @%s\n' % (ignore_recent, bucket))
t.read_until('OK')
t.write('quit\n')
t.close()
def _gc_status(self):
t = telnetlib.Telnet("127.0.0.1", self.backend1.port)
t.write('optimize_stat\n')
out = t.read_until('\n')
t.write('quit\n')
t.close()
return out.strip("\r\n")
def _gen_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def _delete_data(self, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def _check_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
class TestGCBase(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
def _start_gc(self, ignore_recent=0, bucket=None):
""" bucket must be in 0 or 00 string """
if bucket is not None:
assert isinstance(bucket, basestring) and len(bucket) <= 2
t = telnetlib.Telnet("127.0.0.1", self.backend1.port)
if bucket is None:
t.write('flush_all {}\n'.format(ignore_recent))
else:
t.write('flush_all %s @%s\n' % (ignore_recent, bucket))
t.read_until('OK')
t.write('quit\n')
t.close()
def _gc_status(self):
t = telnetlib.Telnet("127.0.0.1", self.backend1.port)
t.write('optimize_stat\n')
out = t.read_until('\n')
t.write('quit\n')
t.close()
return out.strip("\r\n")
def _gen_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.set(key, data):
return self.fail("fail to set %s" % (key))
def _delete_data(self, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
if not store.delete(key):
return self.fail("fail to delete %s" % (key))
def _check_data(self, data, prefix='', loop_num=10 * 1024):
store = MCStore(self.backend1_addr)
for key in self.backend1.generate_key(prefix=prefix, count=loop_num):
try:
self.assertEqual(store.get(key), data)
except Exception, e:
return self.fail("fail to check key %s: %s" % (key, str(e)))
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.last_pos = 0
self.last_size = 0
class TestGenerateData(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
def test_gen_data(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
self.assert_(store.set("largekey", string_large))
self.assert_(store.get("largekey") == string_large)
loop_num = 16 * 1024
for i in xrange(loop_num):
key = "test%s" % (i)
if not store.set(key, 1):
print "failed to set %s" % (key)
return self.fail("fail")
if not store.set(key, 2):
print "failed to set %s" % (key)
return self.fail("fail")
print "done set"
for i in xrange(loop_num):
key = "test%s" % (i)
try:
self.assertEqual(store.get(key), 2)
except Exception, e:
print key, "error", e
return self.fail("fail")
print "done get"
self.backend1.stop()
print "stopped"
self.backend1.start()
print "started"
store = MCStore(self.backend1_addr)
for i in xrange(loop_num):
key = "test%s" % (i)
try:
self.assertEqual(store.get(key), 2)
except Exception, e:
print key, "error", e
return self.fail("fail")
class TestKeyVersion(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def _get_version(self, store, key):
meta = store.get("?" + key)
if meta:
return int(meta.split()[0])
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_version(store, key), 1)
store.set_raw(key, "bbb", rev=3)
self.assertEqual(self._get_version(store, key), 3)
store.set_raw(key, "bbb", rev=4)
self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key2"
store.set(key, "aaa")
self.assertEqual(self._get_version(store, key), 1)
store.delete(key)
self.assertEqual(self._get_version(store, key), None)
store.set(key, "bbb")
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_version(store, key), 3)
def tearDown(self):
self.backend1.stop()
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, accesslog=False, max_data_size=10)
class Test4(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test4.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test4.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr +
' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port':
7905,
'webport':
7908,
'threads':
8,
'n':
3,
'w':
2,
'r':
1,
'buckets':
16,
'slow':
100,
'listen':
'0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def _iptable_block(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -I INPUT 1 -i lo -p tcp --dport %s -j DROP" % (
port)
print cmd
os.system(cmd)
def _iptable_unblock(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -i lo -D INPUT -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test3(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test3.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test3.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test3(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test3.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test3.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr +
' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port':
7905,
'webport':
7908,
'threads':
8,
'n':
3,
'w':
2,
'r':
1,
'buckets':
16,
'slow':
100,
'listen':
'0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (
i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class TestKeyVersion(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def _get_meta(self, store, key):
meta = store.get("??" + key)
if meta:
meta = meta.split()
assert (len(meta) == 7)
return tuple([int(meta[i]) for i in [0, -2, -1]])
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.last_pos = 0
self.last_size = 0
def append(self, size):
self.last_pos += self.last_size
self.last_size = size
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.set_raw(key, "bbb", rev=3)
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
store.set_raw(key, "bbb", rev=4)
self.assertEqual(
self._get_meta(store, key),
(4, 0, self.last_pos)) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(
self._get_meta(store, key),
(4, 0, self.last_pos)) # current behavior will raise version
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.delete(key)
self.append(256)
self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos))
store.set(key, "bbb")
self.append(256)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def _test_compress(self, overflow):
self.backend1.start()
value = string.letters
store = MCStore(self.backend1_addr)
compressed_value = zlib.compress(value, 0)
key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0))
value_easy_compress = 'v' * len(compressed_value)
assert (store.set(key, value_easy_compress))
assert (store.get(key) == value_easy_compress)
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert (store.set_raw(key, compressed_value, flag=0x00000010))
assert (store.get(key) == value)
self.append(512 if overflow else 256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
assert (store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_compress_257(self):
self._test_compress(True)
def test_compress_256(self):
self._test_compress(False)
def test_special_key(self):
self.backend1.start()
kvs = [('a' * 250, 1), ("a", range(1000))]
store = MCStore(self.backend1_addr)
for k, v in kvs:
assert (store.set(k, v))
assert (v == store.get(k))
self.backend1.stop()
self.backend1.start()
store = MCStore(self.backend1_addr)
for (k, v) in kvs:
assert (v == store.get(k))
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert (store.set(key, string_large))
assert (store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert (store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_restart(self):
pass
def tearDown(self):
self.backend1.stop()
class Test1(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test1.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test1.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend1.start()
self.backend2.start()
self.backend3.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test1(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
print "down backend2, proxy.get should be ok"
self.backend2.stop()
proxy.delete('key2')
self.assert_(not proxy.exists('key2'))
self.assert_(not MCStore(self.backend1_addr).exists('key2'))
self.assert_(not MCStore(self.backend3_addr).exists('key2'))
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self.assert_(proxy.exists('key2'))
self.assert_(MCStore(self.backend3_addr).exists('key2'))
self.assert_(MCStore(self.backend1_addr).exists('key2'))
self._assert_data(self.proxy_addr, 'key2', data2)
self._assert_data(self.backend1_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
print "down backend1, proxy.get/set should fail"
self.backend1.stop()
self.assertEqual(proxy.get('key1'), data1)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
with self.assertRaises(Exception) as exc:
proxy.set('key2', data2)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=1)
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=2)
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901, accesslog=False, max_data_size=10)
class TestKeyVersion(TestBeansdbBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
def _get_meta(self, store, key):
meta = store.get("??" + key)
if meta:
meta = meta.split()
assert(len(meta) == 7)
return tuple([int(meta[i]) for i in [0, -2, -1]])
def setUp(self):
self._clear_dir()
self._init_dir()
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.last_pos = 0
self.last_size = 0
def append(self, size):
self.last_pos += self.last_size
self.last_size = size
def test_set_verion(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(store.get(key), "aaa")
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.set_raw(key, "bbb", rev=3)
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
store.set_raw(key, "bbb", rev=4)
self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
store.set_raw(key, "ccc", rev=2)
self.assertEqual(store.get(key), "bbb")
self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version
def test_delete_version(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "key1"
store.set(key, "aaa")
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
store.delete(key)
self.append(256)
self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos))
store.set(key, "bbb")
self.append(256)
self.assertEqual(store.get(key), 'bbb')
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def _test_compress(self, overflow):
self.backend1.start()
value = string.letters
store = MCStore(self.backend1_addr)
compressed_value = zlib.compress(value, 0)
key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0))
value_easy_compress = 'v'* len(compressed_value)
assert(store.set(key, value_easy_compress))
assert(store.get(key) == value_easy_compress)
self.append(256)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert(store.set_raw(key, compressed_value, flag = 0x00000010))
assert(store.get(key) == value)
self.append(512 if overflow else 256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
assert(store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos))
def test_compress_257(self):
self._test_compress(True)
def test_compress_256(self):
self._test_compress(False)
def test_special_key(self):
self.backend1.start()
kvs = [('a'*250, 1), ("a", range(1000))]
store = MCStore(self.backend1_addr)
for k,v in kvs:
assert(store.set(k, v))
assert(v == store.get(k))
self.backend1.stop()
self.backend1.start()
store = MCStore(self.backend1_addr)
for (k,v) in kvs:
assert(v == store.get(k))
def test_big_value(self):
self.backend1.start()
store = MCStore(self.backend1_addr)
key = "largekey"
size = 10 * 1024 * 1024
rsize = (((size + len(key) + 24) >> 8) + 1) << 8
string_large = random_string(size)
assert(store.set(key, string_large))
assert(store.get(key) == string_large)
self.append(rsize)
self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos))
assert(store.set(key, 'aaa'))
self.append(256)
self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos))
def test_restart(self):
pass
def tearDown(self):
self.backend1.stop()
class Test2(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test2.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test2.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test2(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self._assert_data(self.backend4_addr, 'key1', None, "temporary node should not have the key when all primary nodes is good")
proxy.delete('key2')
print "down backend1 and backend2, proxy.get should be ok"
self.backend1.stop()
self.backend2.stop()
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self._assert_data(self.proxy_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
#"temporary node should have the key when primary nodes < 2"
self._assert_data(self.backend4_addr, 'key2', data2)
print "test delete under bad sistuation, will raise error according to current behavior"
with self.assertRaises(Exception) as exc:
proxy.delete('key2')
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
print "start backend2, (backend1 still down), test delete"
self.backend2.start()
time.sleep(10)
proxy.delete('key2')
self._assert_data(self.proxy_addr, 'key2', None)
self._assert_data(self.backend2_addr, 'key2', None)
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test4(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test4.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test4.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def _iptable_block(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -I INPUT 1 -i lo -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def _iptable_unblock(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -i lo -D INPUT -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class TestLogRotate(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_testlog.log')
accesslog_bak = os.path.join(data_base_path, 'beansproxy_testlog.log_bak')
errorlog = os.path.join(data_base_path, 'beansproxy_error_testlog.log')
errorlog_bak = os.path.join(data_base_path, 'beansproxy_error_testlog.log_bak')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend1.start()
self.backend2.start()
self.backend3.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test1(self):
data1 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print cmd
print subprocess.check_output(cmd, shell=True)
print "move log"
if os.path.exists(self.accesslog_bak):
os.remove(self.accesslog_bak)
if os.path.exists(self.errorlog_bak):
os.remove(self.errorlog_bak)
os.rename(self.accesslog, self.accesslog_bak)
os.rename(self.errorlog, self.errorlog_bak)
print "write more data to see if new log not exists"
data1 = random_string(10)
proxy.set('key1', data1)
self.assert_(not os.path.exists(self.accesslog))
self.assert_(not os.path.exists(self.errorlog))
time.sleep(5)
print "send SIGINT signal, should re-open log file"
os.kill(self.proxy_p.pid, signal.SIGINT)
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print subprocess.check_output(cmd, shell=True)
s = os.stat(self.accesslog)
self.assert_(os.path.exists(self.accesslog))
self.assert_(os.path.exists(self.errorlog))
print "see if write to new accesslog"
proxy.get('key1')
time.sleep(1)
s_new = os.stat(self.accesslog)
print s_new.st_size, s.st_size
self.assert_(s_new.st_size > s.st_size)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
