class Test4(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test4.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test4.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def _iptable_block(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -I INPUT 1 -i lo -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def _iptable_unblock(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -i lo -D INPUT -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test3(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test3.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test3.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test4(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test4.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test4.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr +
' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port':
7905,
'webport':
7908,
'threads':
8,
'n':
3,
'w':
2,
'r':
1,
'buckets':
16,
'slow':
100,
'listen':
'0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def _iptable_block(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -I INPUT 1 -i lo -p tcp --dport %s -j DROP" % (
port)
print cmd
os.system(cmd)
def _iptable_unblock(self, addr):
ip, port = addr.split(":")
cmd = "sudo iptables -i lo -D INPUT -p tcp --dport %s -j DROP" % (port)
print cmd
os.system(cmd)
def test_iptable_silence(self):
""" test wether a node slow will affect response time """
proxy = BeansDBProxy([self.proxy_addr])
key3 = 'key3'
i = 0
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
i = 0
self._iptable_block(self.backend1_addr)
start_time = time.time()
for i in range(20000):
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
self.assertEqual(proxy.get(key3), data3)
print "avg get&set time", (time.time() - start_time) / 2000
self._iptable_unblock(self.backend1_addr)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test1(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test1.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test1.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend1.start()
self.backend2.start()
self.backend3.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test1(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
print "down backend2, proxy.get should be ok"
self.backend2.stop()
proxy.delete('key2')
self.assert_(not proxy.exists('key2'))
self.assert_(not MCStore(self.backend1_addr).exists('key2'))
self.assert_(not MCStore(self.backend3_addr).exists('key2'))
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self.assert_(proxy.exists('key2'))
self.assert_(MCStore(self.backend3_addr).exists('key2'))
self.assert_(MCStore(self.backend1_addr).exists('key2'))
self._assert_data(self.proxy_addr, 'key2', data2)
self._assert_data(self.backend1_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
print "down backend1, proxy.get/set should fail"
self.backend1.stop()
self.assertEqual(proxy.get('key1'), data1)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
with self.assertRaises(Exception) as exc:
proxy.set('key2', data2)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
class Test3(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test3.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test3.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr +
' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port':
7905,
'webport':
7908,
'threads':
8,
'n':
3,
'w':
2,
'r':
1,
'buckets':
16,
'slow':
100,
'listen':
'0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (
i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class Test2(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
backend4_addr = 'localhost:57904'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_test2.log')
errorlog = os.path.join(data_base_path, 'beansproxy_error_test2.log')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend4 = BeansdbInstance(self.data_base_path, 57904)
self.backend1.start()
self.backend2.start()
self.backend3.start()
self.backend4.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend4_addr + ' -F -E -D -C -B -A -9 -8 -7 -6 -5 -4 -3 -2 -1 -0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test2(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self._assert_data(self.backend4_addr, 'key1', None, "temporary node should not have the key when all primary nodes is good")
proxy.delete('key2')
print "down backend1 and backend2, proxy.get should be ok"
self.backend1.stop()
self.backend2.stop()
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self._assert_data(self.proxy_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
#"temporary node should have the key when primary nodes < 2"
self._assert_data(self.backend4_addr, 'key2', data2)
print "test delete under bad sistuation, will raise error according to current behavior"
with self.assertRaises(Exception) as exc:
proxy.delete('key2')
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
print "start backend2, (backend1 still down), test delete"
self.backend2.start()
time.sleep(10)
proxy.delete('key2')
self._assert_data(self.proxy_addr, 'key2', None)
self._assert_data(self.backend2_addr, 'key2', None)
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend4.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
self.backend4.clean()
class TestLogRotate(TestBeanseyeBase):
proxy_addr = 'localhost:7905'
backend1_addr = 'localhost:57901'
backend2_addr = 'localhost:57902'
backend3_addr = 'localhost:57903'
data_base_path = os.path.join("/tmp", "beanseye_test")
accesslog = os.path.join(data_base_path, 'beansproxy_testlog.log')
accesslog_bak = os.path.join(data_base_path, 'beansproxy_testlog.log_bak')
errorlog = os.path.join(data_base_path, 'beansproxy_error_testlog.log')
errorlog_bak = os.path.join(data_base_path, 'beansproxy_error_testlog.log_bak')
def setUp(self):
self._init_dir()
if os.path.exists(self.accesslog):
os.remove(self.accesslog)
if os.path.exists(self.errorlog):
os.remove(self.errorlog)
self.backend1 = BeansdbInstance(self.data_base_path, 57901)
self.backend2 = BeansdbInstance(self.data_base_path, 57902)
self.backend3 = BeansdbInstance(self.data_base_path, 57903)
self.backend1.start()
self.backend2.start()
self.backend3.start()
# start 3 backend, no additional temporary node
proxy_conf = {
'servers': [
self.backend1_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend2_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
self.backend3_addr + ' F E D C B A 9 8 7 6 5 4 3 2 1 0',
],
'port': 7905,
'webport': 7908,
'threads': 8,
'n': 3,
'w': 2,
'r': 1,
'buckets': 16,
'slow': 100,
'listen': '0.0.0.0',
'proxies': [self.proxy_addr],
}
self.proxy_p = self._start_proxy(proxy_conf)
def test1(self):
data1 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print cmd
print subprocess.check_output(cmd, shell=True)
print "move log"
if os.path.exists(self.accesslog_bak):
os.remove(self.accesslog_bak)
if os.path.exists(self.errorlog_bak):
os.remove(self.errorlog_bak)
os.rename(self.accesslog, self.accesslog_bak)
os.rename(self.errorlog, self.errorlog_bak)
print "write more data to see if new log not exists"
data1 = random_string(10)
proxy.set('key1', data1)
self.assert_(not os.path.exists(self.accesslog))
self.assert_(not os.path.exists(self.errorlog))
time.sleep(5)
print "send SIGINT signal, should re-open log file"
os.kill(self.proxy_p.pid, signal.SIGINT)
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print subprocess.check_output(cmd, shell=True)
s = os.stat(self.accesslog)
self.assert_(os.path.exists(self.accesslog))
self.assert_(os.path.exists(self.errorlog))
print "see if write to new accesslog"
proxy.get('key1')
time.sleep(1)
s_new = os.stat(self.accesslog)
print s_new.st_size, s.st_size
self.assert_(s_new.st_size > s.st_size)
def tearDown(self):
stop_svc(self.proxy_p)
self.backend1.stop()
self.backend2.stop()
self.backend3.stop()
self.backend1.clean()
self.backend2.clean()
self.backend3.clean()
