def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def test2(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self._assert_data(self.backend4_addr, 'key1', None, "temporary node should not have the key when all primary nodes is good")
proxy.delete('key2')
print "down backend1 and backend2, proxy.get should be ok"
self.backend1.stop()
self.backend2.stop()
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self._assert_data(self.proxy_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
#"temporary node should have the key when primary nodes < 2"
self._assert_data(self.backend4_addr, 'key2', data2)
print "test delete under bad sistuation, will raise error according to current behavior"
with self.assertRaises(Exception) as exc:
proxy.delete('key2')
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
print "start backend2, (backend1 still down), test delete"
self.backend2.start()
time.sleep(10)
proxy.delete('key2')
self._assert_data(self.proxy_addr, 'key2', None)
self._assert_data(self.backend2_addr, 'key2', None)
self._assert_data(self.backend3_addr, 'key2', None)
self._assert_data(self.backend4_addr, 'key2', None)
def test3(self):
""" test wether will fallback only down 2 primary node """
self.skipTest("disable acorrding to current behavior")
return
proxy = BeansDBProxy([self.proxy_addr])
self.backend1.stop()
# self.backend2.stop()
key3 = 'key3'
i = 0
store4 = MCStore(self.backend4_addr)
ts_start = time.time()
fallbacked = False
while i < 2000:
i += 1
data3 = random_string(10)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
# time.sleep(0.1)
data3_ = store4.get(key3)
if data3_ is None:
print "store4 get nothing yet, round=", i
else:
print "fallbacked to store4 after %s tries" % (i)
fallbacked = True
self.assertEqual(data3_, data3)
break
ts_stop = time.time()
if not fallbacked:
self.fail("still not fallback to backend 4")
print "%s seconds passed" % (ts_stop - ts_start)
self.backend1.start()
self.assert_(proxy.exists("key3"))
store1 = MCStore(self.backend1_addr)
self.assert_(store1.get("key3") is None)
data3 = random_string(10)
ts_recover_start = time.time()
i = 0
recovered = False
while i < 2000:
#data3 = random_string(10)
i += 1
# time.sleep(0.1)
proxy.set(key3, data3)
self.assertEqual(proxy.get(key3), data3)
data3_ = store1.get(key3)
if data3_ is None:
print "store1 get nothing yet, round=", i
else:
print "recover to store1 after %s tries, %s sec" % (
i, time.time() - ts_recover_start)
recovered = True
self.assertEqual(data3_, data3)
break
if not recovered:
self.fail("still not fallback to backend 1")
def test1(self):
data1 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print cmd
print subprocess.check_output(cmd, shell=True)
print "move log"
if os.path.exists(self.accesslog_bak):
os.remove(self.accesslog_bak)
if os.path.exists(self.errorlog_bak):
os.remove(self.errorlog_bak)
os.rename(self.accesslog, self.accesslog_bak)
os.rename(self.errorlog, self.errorlog_bak)
print "write more data to see if new log not exists"
data1 = random_string(10)
proxy.set('key1', data1)
self.assert_(not os.path.exists(self.accesslog))
self.assert_(not os.path.exists(self.errorlog))
time.sleep(5)
print "send SIGINT signal, should re-open log file"
os.kill(self.proxy_p.pid, signal.SIGINT)
cmd = "ls -l /proc/%s/fd" % (self.proxy_p.pid)
print subprocess.check_output(cmd, shell=True)
s = os.stat(self.accesslog)
self.assert_(os.path.exists(self.accesslog))
self.assert_(os.path.exists(self.errorlog))
print "see if write to new accesslog"
proxy.get('key1')
time.sleep(1)
s_new = os.stat(self.accesslog)
print s_new.st_size, s.st_size
self.assert_(s_new.st_size > s.st_size)
def test1(self):
data1 = random_string(10)
data2 = random_string(10)
time.sleep(1)
print "test normal write"
proxy = BeansDBProxy([self.proxy_addr])
proxy.delete('key1')
proxy.set('key1', data1)
self._assert_data(self.backend1_addr, 'key1', data1)
self._assert_data(self.backend2_addr, 'key1', data1)
self._assert_data(self.backend3_addr, 'key1', data1)
self.assert_(MCStore(self.backend2_addr).exists('key1'))
print "down backend2, proxy.get should be ok"
self.backend2.stop()
proxy.delete('key2')
self.assert_(not proxy.exists('key2'))
self.assert_(not MCStore(self.backend1_addr).exists('key2'))
self.assert_(not MCStore(self.backend3_addr).exists('key2'))
proxy.set('key2', data2)
self.assertEqual(proxy.get('key2'), data2)
self.assert_(proxy.exists('key2'))
self.assert_(MCStore(self.backend3_addr).exists('key2'))
self.assert_(MCStore(self.backend1_addr).exists('key2'))
self._assert_data(self.proxy_addr, 'key2', data2)
self._assert_data(self.backend1_addr, 'key2', data2)
with self.assertRaises(Exception) as exc:
MCStore(self.backend2_addr).get('key2')
self._assert_data(self.backend3_addr, 'key2', data2)
print "down backend1, proxy.get/set should fail"
self.backend1.stop()
self.assertEqual(proxy.get('key1'), data1)
with self.assertRaises(Exception) as exc:
MCStore(self.backend1_addr).get('key2')
MCStore(self.backend2_addr).get('key2')
with self.assertRaises(Exception) as exc:
proxy.set('key2', data2)
def _assert_data(self, addr, key, data, mesg=None):
store = MCStore(addr)
self.assertEqual(store.get(key), data, mesg)
