def verify_dump_exists_after_n_crashes(self): n = 5 vbucket = 0 index = 0 nodeA = self.servers[index] crashes = 0 logp = NSLogPoller(index) logp.start() self.load_docs(nodeA, 100000) # watch for mcd restarted logp.setMcEventFlag(True) # kill mc n times for i in range(n): rc = self.kill_memcached(0) assert rc, "Did not kill mc" # get restart msg msg = logp.getEventQItem() assert msg crashes += 1 logp.setMcEventFlag(False) while True: dmp_path = logp.getDumpQItem() if dmp_path is None: break crashes -= 1 assert crashes == 0, "Did not report all crashes"
def verify_dump_exists_after_n_crashes(self): n = 5 vbucket = 0 index = 0 nodeA = self.servers[index] crashes = 0 logp = NSLogPoller(index) logp.start() self.load_docs(nodeA, 100000) # watch for mcd restarted logp.setMcEventFlag(True) # kill mc n times for i in range(n): rc = self.kill_memcached(0) assert rc, "Did not kill mc" # get restart msg msg = logp.getEventQItem() assert msg crashes += 1 logp.setMcEventFlag(False) while True: dmp_path = logp.getDumpQItem() if dmp_path is None: break crashes -= 1 assert crashes == 0, "Did not report all crashes"
def kill_during_concurrent_loader(self): nodeA = self.servers[0] logp = NSLogPoller(0) logp.start() running = True threads = [] # inline doc loader def loader(): while running: try: self.load_docs(nodeA, 1000) except: pass # async doc loaders for i in range(10): t = threading.Thread(target=loader) t.start() threads.append(t) assert self.kill_memcached(0) dmp_path = logp.getDumpQItem() assert os.access(dmp_path, os.R_OK) running = False [t.join() for t in threads]
def kill_during_concurrent_loader(self): nodeA = self.servers[0] logp = NSLogPoller(0) logp.start() running = True threads = [] # inline doc loader def loader(): while running: try: self.load_docs(nodeA, 1000) except: pass # async doc loaders for i in range(10): t = threading.Thread(target=loader) t.start() threads.append(t) assert self.kill_memcached(0) dmp_path = logp.getDumpQItem() assert os.access(dmp_path, os.R_OK) running = False [t.join() for t in threads]
def verify_dump_exists_various_kills(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill node 0 with sig=6 assert self.kill_memcached(0, sig=6) dmp_path = logp.getDumpQItem() assert dmp_path and os.access(dmp_path, os.R_OK) logp = NSLogPoller(1) logp.start() # SIGSEGV to node1 assert self.kill_memcached(1, sig=11) dmp_path = logp.getDumpQItem() assert dmp_path and os.access(dmp_path, os.R_OK)
def verify_dump_exists_various_kills(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill node 0 with sig=6 assert self.kill_memcached(0, sig=6) dmp_path = logp.getDumpQItem() assert dmp_path and os.access(dmp_path, os.R_OK) logp = NSLogPoller(1) logp.start() # SIGSEGV to node1 assert self.kill_memcached(1, sig=11) dmp_path = logp.getDumpQItem() assert dmp_path and os.access(dmp_path, os.R_OK)
def kill_during_compaction(self): nodeA = self.servers[0] logp = NSLogPoller(0) logp.start() running = True threads = [] def loader(): while running: try: self.load_docs(nodeA, 1000) except: pass def compact(): rest = RestConnection(nodeA) rest.compact_bucket() # async doc loaders for i in range(10): t = threading.Thread(target=loader) t.start() threads.append(t) # watch for compaction start logp.setCompactEventFlag(True) # async compact t = threading.Thread(target=compact) t.start() # verify compaction event did happen assert logp.getEventQItem() logp.setCompactEventFlag(False) # kill mc assert self.kill_memcached(0) # stop data loading running = False [t.join() for t in threads] # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert dmp_path and os.access(dmp_path, os.R_OK)
def kill_during_compaction(self): nodeA = self.servers[0] logp = NSLogPoller(0) logp.start() running = True threads = [] def loader(): while running: try: self.load_docs(nodeA, 1000) except: pass def compact(): rest = RestConnection(nodeA) rest.compact_bucket() # async doc loaders for i in range(10): t = threading.Thread(target=loader) t.start() threads.append(t) # watch for compaction start logp.setCompactEventFlag(True) # async compact t = threading.Thread(target=compact) t.start() # verify compaction event did happen assert logp.getEventQItem() logp.setCompactEventFlag(False) # kill mc assert self.kill_memcached(0) # stop data loading running = False [t.join() for t in threads] # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert dmp_path and os.access(dmp_path, os.R_OK)
def verify_dump_exists_after_crash(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill memcahced assert self.kill_memcached(0) # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert dmp_path and os.access(dmp_path, os.R_OK)
def verify_dump_exists_after_crash(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill memcahced assert self.kill_memcached(0) # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert dmp_path and os.access(dmp_path, os.R_OK)
def verify_mini_dmp_to_core(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill memcahced assert self.kill_memcached(0) # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert os.access(dmp_path, os.R_OK) f_core = self.dmp_to_core(dmp_path) assert os.access(f_core, os.R_OK) assert self.verify_core(f_core)
def verify_mini_dmp_to_core(self): vbucket = 0 index = 0 nodeA = self.servers[index] # load some documents self.load_docs(nodeA, 1000) # start log poller logp = NSLogPoller(index) logp.start() # kill memcahced assert self.kill_memcached(0) # get dmp_path from log poller dmp_path = logp.getDumpQItem() # verify a coresponding dmp exists on filesystem assert os.access(dmp_path, os.R_OK) f_core = self.dmp_to_core(dmp_path) assert os.access(f_core, os.R_OK) assert self.verify_core(f_core)