def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.mongo_doc._search()) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: result_set_1 = self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue while(len(self.mongo_doc._search()) != 2): time.sleep(1) result_set_1 = self.mongo_doc._search() result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_2), 2) #make sure pauling is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.mongo_doc._search() self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_rollback(self): """Tests rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.mongo_doc._search()) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: result_set_1 = self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: time.sleep(1) count += 1 if count >= 60: sys.exit(1) continue while (len(self.mongo_doc._search()) != 2): time.sleep(1) result_set_1 = self.mongo_doc._search() result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_2), 2) #make sure pauling is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.mongo_doc._search() self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ while len(self.mongo_doc._search()) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) while len(self.mongo_doc._search()) != NUMBER_OF_DOC_DIRS: time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.mongo_doc._search()) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len(self.mongo_doc._search()) != NUMBER_OF_DOC_DIRS): time.sleep(5) result_set_1 = self.mongo_doc._search() self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Rollback is performed like before, but with more documents. """ while len(self.synchronizer._search()) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}, safe=True) while len(self.synchronizer._search()) != NUMBER_OF_DOC_DIRS: time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert({'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.synchronizer._search()) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.synchronizer._search() i = 0 for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len(self.synchronizer._search()) != NUMBER_OF_DOC_DIRS): time.sleep(5) result_set_1 = self.synchronizer._search() self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert( {'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too ' 'many times in test_rollback') time.sleep(1) continue while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1)
def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert({'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too ' 'many times in test_rollback') time.sleep(1) continue while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1)
def test_rollback(self): """Tests rollback. Rollback is performed by inserting one document, killing primary, inserting another doc, killing secondary, and then restarting both. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) self.conn['test']['test'].insert({'name': 'paul'}, safe=True) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: self.conn['test']['test'].insert({'name': 'pauline'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert failed too' ' many times in test_rollback') time.sleep(1) continue while (len(self.synchronizer._search()) != 2): time.sleep(1) result_set_1 = self.synchronizer._search() result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) time.sleep(2) result_set_1 = self.synchronizer._search() self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) search = self.mongo_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] wait_for(lambda : admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert({'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) wait_for(lambda : sum(1 for _ in self.mongo_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) db_admin = primary_conn['admin'] wait_for(lambda : db_admin.command("isMaster")['ismaster']) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) search = self.mongo_doc._search condition = lambda : sum(1 for _ in search()) == NUMBER_OF_DOC_DIRS wait_for(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), NUMBER_OF_DOC_DIRS) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), NUMBER_OF_DOC_DIRS)
def test_single_target(self): """Test with a single replication target""" self.opman.start() # Insert first document with primary up self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn assert_soon(lambda: secondary["test"]["mc"].count() == 1, "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc(self.primary_p, destroy=False) # Wait for the secondary to be promoted while not secondary["admin"].command("isMaster")["ismaster"]: time.sleep(1) # Insert another document. This will be rolled back later retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": 1}) self.assertEqual(secondary["test"]["mc"].count(), 2) # Wait for replication to doc manager assert_soon(lambda: len(self.opman.doc_managers[0]._search()) == 2, "not all writes were replicated to doc manager") # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] assert_soon(lambda: primary_admin.command("isMaster")["ismaster"], "restarted primary never resumed primary status") restart_mongo_proc(self.secondary_p) assert_soon(lambda: retry_until_ok(secondary.admin.command, 'replSetGetStatus')['myState'] == 2, "restarted secondary never resumed secondary status") assert_soon(lambda: retry_until_ok(self.main_conn.test.mc.find().count) > 0, "documents not found after primary/secondary restarted") # Only first document should exist in MongoDB self.assertEqual(self.main_conn["test"]["mc"].count(), 1) self.assertEqual(self.main_conn["test"]["mc"].find_one()["i"], 0) # Same case should hold for the doc manager doc_manager = self.opman.doc_managers[0] self.assertEqual(len(doc_manager._search()), 1) self.assertEqual(doc_manager._search()[0]["i"], 0) # cleanup self.opman.join()
def test_single_target(self): """Test with a single replication target""" self.opman.start() # Insert first document with primary up self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn assert_soon(lambda: secondary["test"]["mc"].count() == 1, "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc(self.primary_p, destroy=False) # Wait for the secondary to be promoted assert_soon(lambda: secondary["admin"].command("isMaster")["ismaster"]) # Insert another document. This will be rolled back later retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": 1}) self.assertEqual(secondary["test"]["mc"].count(), 2) # Wait for replication to doc manager assert_soon(lambda: len(self.opman.doc_managers[0]._search()) == 2, "not all writes were replicated to doc manager") # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] assert_soon(lambda: primary_admin.command("isMaster")["ismaster"], "restarted primary never resumed primary status") restart_mongo_proc(self.secondary_p) assert_soon( lambda: retry_until_ok(secondary.admin.command, 'replSetGetStatus' )['myState'] == 2, "restarted secondary never resumed secondary status") assert_soon( lambda: retry_until_ok(self.main_conn.test.mc.find().count) > 0, "documents not found after primary/secondary restarted") # Only first document should exist in MongoDB self.assertEqual(self.main_conn["test"]["mc"].count(), 1) self.assertEqual(self.main_conn["test"]["mc"].find_one()["i"], 0) # Same case should hold for the doc manager doc_manager = self.opman.doc_managers[0] self.assertEqual(len(doc_manager._search()), 1) self.assertEqual(doc_manager._search()[0]["i"], 0) # cleanup self.opman.join()
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self. conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn['admin'] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. Strategy for rollback is the same as before. """ for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) assert_soon(lambda: sum(1 for _ in self.mongo_doc._search()) == self.conn['test']['test'].find().count()) result_set_1 = self.mongo_doc._search() for item in result_set_1: if 'Pauline' in item['name']: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], result_set_2['_id']) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) db_admin = primary_conn['admin'] assert_soon(lambda: db_admin.command("isMaster")['ismaster']) restart_mongo_proc(self.secondary_p) search = self.mongo_doc._search condition = lambda: sum(1 for _ in search()) == STRESS_COUNT assert_soon(condition) result_set_1 = list(self.mongo_doc._search()) self.assertEqual(len(result_set_1), STRESS_COUNT) for item in result_set_1: self.assertTrue('Paul' in item['name']) find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), STRESS_COUNT)
def test_stressed_rollback(self): """Test stressed rollback with a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search( '*:*', rows=STRESS_COUNT)) == STRESS_COUNT) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) collection_size = self.conn['test']['test'].find().count() assert_soon(lambda: sum(1 for _ in self.solr_conn.search( '*:*', rows=STRESS_COUNT * 2)) == collection_size) result_set_1 = self.solr_conn.search('Pauline', rows=STRESS_COUNT * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) assert_soon(lambda: sum(1 for _ in self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2)) == 0) result_set_1 = list( self.solr_conn.search('Pauline', rows=STRESS_COUNT * 2)) self.assertEqual(len(result_set_1), 0) result_set_2 = list( self.solr_conn.search('Paul', rows=STRESS_COUNT * 2)) self.assertEqual(len(result_set_2), STRESS_COUNT)
def test_deletions(self): """Test rolling back 'd' operations""" self.opman.start() # Insert a document, wait till it replicates to secondary self.main_conn["test"]["mc"].insert({"i": 0}) self.main_conn["test"]["mc"].insert({"i": 1}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 2) assert_soon(lambda: self.secondary_conn["test"]["mc"].count() == 2, "first write didn't replicate to secondary") # Kill the primary, wait for secondary to be promoted kill_mongo_proc(self.primary_p, destroy=False) assert_soon(lambda: self.secondary_conn["admin"].command("isMaster")[ "ismaster"]) # Delete first document retry_until_ok(self.main_conn["test"]["mc"].remove, {"i": 0}) self.assertEqual(self.secondary_conn["test"]["mc"].count(), 1) # Wait for replication to doc manager assert_soon(lambda: len(self.opman.doc_managers[0]._search()) == 1, "delete was not replicated to doc manager") # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] assert_soon(lambda: primary_admin.command("isMaster")["ismaster"], "restarted primary never resumed primary status") restart_mongo_proc(self.secondary_p) assert_soon( lambda: retry_until_ok(self.secondary_conn.admin.command, 'replSetGetStatus')['myState'] == 2, "restarted secondary never resumed secondary status") # Both documents should exist in mongo assert_soon( lambda: retry_until_ok(self.main_conn["test"]["mc"].count) == 2) # Both document should exist in doc manager doc_manager = self.opman.doc_managers[0] docs = list(doc_manager._search()) self.assertEqual(len(docs), 2, "Expected two documents, but got %r" % docs) self.opman.join()
def test_deletions(self): """Test rolling back 'd' operations""" self.opman.start() # Insert a document, wait till it replicates to secondary self.main_conn["test"]["mc"].insert({"i": 0}) self.main_conn["test"]["mc"].insert({"i": 1}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 2) assert_soon(lambda: self.secondary_conn["test"]["mc"].count() == 2, "first write didn't replicate to secondary") # Kill the primary, wait for secondary to be promoted kill_mongo_proc(self.primary_p, destroy=False) assert_soon(lambda: self.secondary_conn["admin"] .command("isMaster")["ismaster"]) # Delete first document retry_until_ok(self.main_conn["test"]["mc"].remove, {"i": 0}) self.assertEqual(self.secondary_conn["test"]["mc"].count(), 1) # Wait for replication to doc manager assert_soon(lambda: len(self.opman.doc_managers[0]._search()) == 1, "delete was not replicated to doc manager") # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] assert_soon(lambda: primary_admin.command("isMaster")["ismaster"], "restarted primary never resumed primary status") restart_mongo_proc(self.secondary_p) assert_soon(lambda: retry_until_ok(self.secondary_conn.admin.command, 'replSetGetStatus')['myState'] == 2, "restarted secondary never resumed secondary status") # Both documents should exist in mongo assert_soon(lambda: retry_until_ok( self.main_conn["test"]["mc"].count) == 2) # Both document should exist in doc manager doc_manager = self.opman.doc_managers[0] docs = list(doc_manager._search()) self.assertEqual(len(docs), 2, "Expected two documents, but got %r" % docs) self.opman.join()
def test_rollback(self): """Test behavior during a MongoDB rollback. We force a rollback by adding a doc, killing the primary, adding another doc, killing the new primary, and then restarting both. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) condition1 = lambda: self.conn['test']['test'].find( {'name': 'paul'}).count() == 1 condition2 = lambda: self._count() == 1 assert_soon(condition1) assert_soon(condition2) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin = new_primary_conn['admin'] assert_soon(lambda: admin.command("isMaster")['ismaster']) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon(lambda: self._count() == 2) result_set_1 = list(self._search()) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 2) #make sure pauline is there for item in result_set_1: if item['name'] == 'pauline': self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = list(self._search()) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['name'], 'paul') find_cursor = retry_until_ok(self.conn['test']['test'].find) self.assertEqual(retry_until_ok(find_cursor.count), 1)
def test_stressed_rollback(self): """Test stressed rollback with a large number of documents""" for i in range(0, STRESS_COUNT): self.conn["test"]["test"].insert({"name": "Paul " + str(i)}) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*", rows=STRESS_COUNT)) == STRESS_COUNT) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn["admin"] while admin_db.command("isMaster")["ismaster"] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn["test"]["test"].insert({"name": "Pauline " + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) collection_size = self.conn["test"]["test"].find().count() assert_soon(lambda: sum(1 for _ in self.solr_conn.search("*:*", rows=STRESS_COUNT * 2)) == collection_size) result_set_1 = self.solr_conn.search("Pauline", rows=STRESS_COUNT * 2, sort="_id asc") for item in result_set_1: result_set_2 = self.conn["test"]["test"].find_one({"name": item["name"]}) self.assertEqual(item["_id"], str(result_set_2["_id"])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn["admin"].command("isMaster")["ismaster"] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) assert_soon(lambda: sum(1 for _ in self.solr_conn.search("Pauline", rows=STRESS_COUNT * 2)) == 0) result_set_1 = list(self.solr_conn.search("Pauline", rows=STRESS_COUNT * 2)) self.assertEqual(len(result_set_1), 0) result_set_2 = list(self.solr_conn.search("Paul", rows=STRESS_COUNT * 2)) self.assertEqual(len(result_set_2), STRESS_COUNT)
def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) while self.conn['test']['test'].find({'name': 'paul'}).count() != 1: time.sleep(1) while len(self.solr_conn.search('*:*')) != 1: time.sleep(1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) while (len(self.solr_conn.search('*:*')) != 2): time.sleep(1) result_set_1 = self.solr_conn.search('pauline') result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(len(result_set_2), 1)
def test_rollback(self): """Tests rollback. We force a rollback by inserting one doc, killing primary, adding another doc, killing the new primary, and restarting both the servers. """ primary_conn = MongoClient(mongo_host, self.primary_p) self.conn['test']['test'].insert({'name': 'paul'}) assert_soon( lambda: self.conn.test.test.find({'name': 'paul'}).count() == 1) assert_soon( lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 1) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) retry_until_ok(self.conn.test.test.insert, {'name': 'pauline'}) assert_soon( lambda: sum(1 for _ in self.solr_conn.search('*:*')) == 2) result_set_1 = list(self.solr_conn.search('pauline')) result_set_2 = self.conn['test']['test'].find_one({'name': 'pauline'}) self.assertEqual(len(result_set_1), 1) for item in result_set_1: self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) time.sleep(2) result_set_1 = self.solr_conn.search('pauline') self.assertEqual(sum(1 for _ in result_set_1), 0) result_set_2 = self.solr_conn.search('paul') self.assertEqual(sum(1 for _ in result_set_2), 1)
def tearDownClass(cls): kill_mongo_proc('localhost', 30000)
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. The rollback is performed the same way as before but with more docs """ self.conn['test']['test'].remove() while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert({'name': 'Paul ' + str(i)}, safe=True) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len( self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('Paul', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)
def test_stressed_rollback(self): """Test stressed rollback with a large number of documents""" for i in range(0, STRESS_COUNT): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}) while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT)) != STRESS_COUNT): time.sleep(1) primary_conn = MongoClient(mongo_host, self.primary_p) kill_mongo_proc(self.primary_p, destroy=False) new_primary_conn = MongoClient(mongo_host, self.secondary_p) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < STRESS_COUNT: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=STRESS_COUNT * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2, sort='_id asc' ) for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(self.secondary_p, destroy=False) restart_mongo_proc(self.primary_p) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) restart_mongo_proc(self.secondary_p) while (len(self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search( 'Pauline', rows=STRESS_COUNT * 2 ) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search( 'Paul', rows=STRESS_COUNT * 2 ) self.assertEqual(len(result_set_2), STRESS_COUNT)
def test_many_targets(self): """Test with several replication targets""" # OplogThread has multiple doc managers doc_managers = [DocManager(), DocManager(), DocManager()] self.opman.doc_managers = doc_managers self.opman.start() # Insert a document into each namespace self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn assert_soon(lambda: secondary["test"]["mc"].count() == 1, "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc(self.primary_p, destroy=False) # Wait for the secondary to be promoted assert_soon(lambda: secondary.admin.command("isMaster")['ismaster'], 'secondary was never promoted') # Insert more documents. This will be rolled back later # Some of these documents will be manually removed from # certain doc managers, to emulate the effect of certain # target systems being ahead/behind others secondary_ids = [] for i in range(1, 10): secondary_ids.append( retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": i})) self.assertEqual(self.secondary_conn["test"]["mc"].count(), 10) # Wait for replication to the doc managers def docmans_done(): for dm in self.opman.doc_managers: if len(dm._search()) != 10: return False return True assert_soon(docmans_done, "not all writes were replicated to doc managers") # Remove some documents from the doc managers to simulate # uneven replication ts = self.opman.doc_managers[0].get_last_doc()['_ts'] for id in secondary_ids[8:]: self.opman.doc_managers[1].remove({ "_id": id, "ns": "test.mc", "_ts": ts }) for id in secondary_ids[2:]: self.opman.doc_managers[2].remove({ "_id": id, "ns": "test.mc", "_ts": ts }) # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] assert_soon(lambda: primary_admin.command("isMaster")['ismaster'], 'restarted primary never resumed primary status') restart_mongo_proc(self.secondary_p) assert_soon( lambda: retry_until_ok(secondary.admin.command, 'replSetGetStatus' )['myState'] == 2, "restarted secondary never resumed secondary status") assert_soon( lambda: retry_until_ok(self.primary_conn.test.mc.find().count) > 0, "documents not found after primary/secondary restarted") # Only first document should exist in MongoDB self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) self.assertEqual(self.primary_conn["test"]["mc"].find_one()["i"], 0) # Give OplogThread some time to catch up time.sleep(10) # Same case should hold for the doc managers for dm in self.opman.doc_managers: self.assertEqual(len(dm._search()), 1) self.assertEqual(dm._search()[0]["i"], 0) self.opman.join()
def test_many_targets(self): """Test with several replication targets""" # OplogThread has multiple doc managers doc_managers = [DocManager(), DocManager(), DocManager()] self.opman.doc_managers = doc_managers self.opman.start() # Insert a document into each namespace self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn self.assertTrue(wait_for(lambda: secondary["test"]["mc"].count() == 1), "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc(self.primary_p, destroy=False) # Wait for the secondary to be promoted while not secondary["admin"].command("isMaster")["ismaster"]: time.sleep(1) # Insert more documents. This will be rolled back later # Some of these documents will be manually removed from # certain doc managers, to emulate the effect of certain # target systems being ahead/behind others secondary_ids = [] for i in range(1, 10): secondary_ids.append( retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": i})) self.assertEqual(self.secondary_conn["test"]["mc"].count(), 10) # Wait for replication to the doc managers def docmans_done(): for dm in self.opman.doc_managers: if len(dm._search()) != 10: return False return True self.assertTrue(wait_for(docmans_done), "not all writes were replicated to doc managers") # Remove some documents from the doc managers to simulate # uneven replication for id in secondary_ids[8:]: self.opman.doc_managers[1].remove({"_id": id}) for id in secondary_ids[2:]: self.opman.doc_managers[2].remove({"_id": id}) # Kill the new primary kill_mongo_proc(self.secondary_p, destroy=False) # Start both servers back up restart_mongo_proc(self.primary_p) primary_admin = self.primary_conn["admin"] while not primary_admin.command("isMaster")["ismaster"]: time.sleep(1) restart_mongo_proc(self.secondary_p) while retry_until_ok(secondary["admin"].command, "replSetGetStatus")["myState"] != 2: time.sleep(1) while retry_until_ok(self.primary_conn["test"]["mc"].find().count) == 0: time.sleep(1) # Only first document should exist in MongoDB self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) self.assertEqual(self.primary_conn["test"]["mc"].find_one()["i"], 0) # Give OplogThread some time to catch up time.sleep(10) # Same case should hold for the doc managers for dm in self.opman.doc_managers: self.assertEqual(len(dm._search()), 1) self.assertEqual(dm._search()[0]["i"], 0) self.opman.join()
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing primary, inserting another doc, killing the new primary, and then restarting both servers. """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) if not start_cluster(sharded=True): self.fail("Shards cannot be added to mongos") test_oplog, primary_conn, solr, mongos = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*:*') safe_mongo_op(mongos['alpha']['foo'].remove, {}) safe_mongo_op(mongos['alpha']['foo'].insert, { '_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie' }) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = { 'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'alpha.foo', '_id': ObjectId('4ff74db3f646462b38000001') } # try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'}) break except OperationFailure: time.sleep(1) count += 1 if count > 60: self.fail('Insert failed too many times in rollback') continue kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) # wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) # wait for secondary to be established admin_db = new_primary_conn['admin'] while admin_db.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['alpha']['foo'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = { 'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'alpha.foo', '_id': obj2 } test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() self.assertEqual(len(results), 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def tearDownClass(cls): kill_mongo_proc(cls.standalone_port)
def test_single_target(self): """Test with a single replication target""" self.opman.start() # Insert first document with primary up self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn self.assertTrue(wait_for(lambda: secondary["test"]["mc"].count() == 1), "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc("localhost", PORTS_ONE["PRIMARY"]) # Wait for the secondary to be promoted while not secondary["admin"].command("isMaster")["ismaster"]: time.sleep(1) # Insert another document. This will be rolled back later retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": 1}) self.assertEqual(secondary["test"]["mc"].count(), 2) # Wait for replication to doc manager c = lambda: len(self.opman.doc_managers[0]._search()) == 2 self.assertTrue(wait_for(c), "not all writes were replicated to doc manager") # Kill the new primary kill_mongo_proc("localhost", PORTS_ONE["SECONDARY"]) # Start both servers back up start_mongo_proc(port=PORTS_ONE['PRIMARY'], repl_set_name="demo-repl", data="/replset1a", log="/replset1a.log", key_file=None) primary_admin = self.primary_conn["admin"] while not primary_admin.command("isMaster")["ismaster"]: time.sleep(1) start_mongo_proc(port=PORTS_ONE['SECONDARY'], repl_set_name="demo-repl", data="/replset1b", log="/replset1b.log", key_file=None) while secondary["admin"].command("replSetGetStatus")["myState"] != 2: time.sleep(1) while retry_until_ok(self.main_conn["test"]["mc"].find().count) == 0: time.sleep(1) # Only first document should exist in MongoDB self.assertEqual(self.main_conn["test"]["mc"].count(), 1) self.assertEqual(self.main_conn["test"]["mc"].find_one()["i"], 0) # Same case should hold for the doc manager doc_manager = self.opman.doc_managers[0] self.assertEqual(len(doc_manager._search()), 1) self.assertEqual(doc_manager._search()[0]["i"], 0) # cleanup self.opman.join()
def test_stressed_rollback(self): """Test stressed rollback with number of documents equal to specified in global variable. The rollback is performed the same way as before but with more docs """ self.conn['test']['test'].remove() while len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != 0: time.sleep(1) for i in range(0, NUMBER_OF_DOC_DIRS): self.conn['test']['test'].insert( {'name': 'Paul ' + str(i)}, safe=True) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS)) != NUMBER_OF_DOC_DIRS): time.sleep(1) primary_conn = Connection(HOSTNAME, int(PORTS_ONE['PRIMARY'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = -1 while count + 1 < NUMBER_OF_DOC_DIRS: try: count += 1 self.conn['test']['test'].insert( {'name': 'Pauline ' + str(count)}, safe=True) except (OperationFailure, AutoReconnect): time.sleep(1) while (len(self.solr_conn.search('*:*', rows=NUMBER_OF_DOC_DIRS * 2)) != self.conn['test']['test'].find().count()): time.sleep(1) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2, sort='_id asc') for item in result_set_1: result_set_2 = self.conn['test']['test'].find_one( {'name': item['name']}) self.assertEqual(item['_id'], str(result_set_2['_id'])) kill_mongo_proc(HOSTNAME, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) while (len(self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2)) != 0): time.sleep(15) result_set_1 = self.solr_conn.search('Pauline', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_1), 0) result_set_2 = self.solr_conn.search('Paul', rows=NUMBER_OF_DOC_DIRS * 2) self.assertEqual(len(result_set_2), NUMBER_OF_DOC_DIRS)
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing primary, inserting another doc, killing the new primary, and then restarting both servers. """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) if not start_cluster(sharded=True): self.fail("Shards cannot be added to mongos") test_oplog, primary_conn, solr, mongos = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*:*') safe_mongo_op(mongos['alpha']['foo'].remove, {}) safe_mongo_op(mongos['alpha']['foo'].insert, {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'alpha.foo', '_id': ObjectId('4ff74db3f646462b38000001')} # try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'}) break except OperationFailure: time.sleep(1) count += 1 if count > 60: self.fail('Insert failed too many times in rollback') continue kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) # wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) # wait for secondary to be established admin_db = new_primary_conn['admin'] while admin_db.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['alpha']['foo'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'alpha.foo', '_id': obj2} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() self.assertEqual(len(results), 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(HOSTNAME, 30000) kill_all()
def tearDownClass(cls): kill_mongo_proc("localhost", 30000)
def test_single_target(self): """Test with a single replication target""" self.opman.start() # Insert first document with primary up self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].find().count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn self.assertTrue(wait_for(lambda: secondary["test"]["mc"].count() == 1), "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc("localhost", PORTS_ONE["PRIMARY"]) # Wait for the secondary to be promoted while not secondary["admin"].command("isMaster")["ismaster"]: time.sleep(1) # Insert another document. This will be rolled back later retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": 1}) self.assertEqual(secondary["test"]["mc"].count(), 2) # Wait for replication to doc manager c = lambda: len(self.opman.doc_managers[0]._search()) == 2 self.assertTrue(wait_for(c), "not all writes were replicated to doc manager") # Kill the new primary kill_mongo_proc("localhost", PORTS_ONE["SECONDARY"]) # Start both servers back up start_mongo_proc( port=PORTS_ONE['PRIMARY'], repl_set_name="demo-repl", data="/replset1a", log="/replset1a.log", key_file=None ) primary_admin = self.primary_conn["admin"] while not primary_admin.command("isMaster")["ismaster"]: time.sleep(1) start_mongo_proc( port=PORTS_ONE['SECONDARY'], repl_set_name="demo-repl", data="/replset1b", log="/replset1b.log", key_file=None ) while secondary["admin"].command("replSetGetStatus")["myState"] != 2: time.sleep(1) while retry_until_ok(self.main_conn["test"]["mc"].find().count) == 0: time.sleep(1) # Only first document should exist in MongoDB self.assertEqual(self.main_conn["test"]["mc"].count(), 1) self.assertEqual(self.main_conn["test"]["mc"].find_one()["i"], 0) # Same case should hold for the doc manager doc_manager = self.opman.doc_managers[0] self.assertEqual(len(doc_manager._search()), 1) self.assertEqual(doc_manager._search()[0]["i"], 0) # cleanup self.opman.join()
def test_rollback(self): """Test the rollback method in a sharded environment Cases: 1. Documents on both shards, rollback on one shard 2. Documents on both shards, rollback on both shards """ self.opman1.start() self.opman2.start() # Insert first documents while primaries are up db_main = self.mongos_conn["test"]["mcsharded"] db_main.insert({"i": 0}, w=2) db_main.insert({"i": 1000}, w=2) self.assertEqual(self.shard1_conn["test"]["mcsharded"].count(), 1) self.assertEqual(self.shard2_conn["test"]["mcsharded"].count(), 1) # Case 1: only one primary goes down, shard1 in this case kill_mongo_proc(self.shard1_prim_p, destroy=False) # Wait for the secondary to be promoted shard1_secondary_admin = self.shard1_secondary_conn["admin"] assert_soon( lambda: shard1_secondary_admin.command("isMaster")["ismaster"]) # Insert another document. This will be rolled back later retry_until_ok(db_main.insert, {"i": 1}) db_secondary1 = self.shard1_secondary_conn["test"]["mcsharded"] db_secondary2 = self.shard2_secondary_conn["test"]["mcsharded"] self.assertEqual(db_secondary1.count(), 2) # Wait for replication on the doc manager # Note that both OplogThreads share the same doc manager c = lambda: len(self.opman1.doc_managers[0]._search()) == 3 assert_soon(c, "not all writes were replicated to doc manager", max_tries=120) # Kill the new primary kill_mongo_proc(self.shard1_scnd_p, destroy=False) # Start both servers back up restart_mongo_proc(self.shard1_prim_p) primary_admin = self.shard1_conn["admin"] c = lambda: primary_admin.command("isMaster")["ismaster"] assert_soon(lambda: retry_until_ok(c)) restart_mongo_proc(self.shard1_scnd_p) secondary_admin = self.shard1_secondary_conn["admin"] c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2 assert_soon(c) query = {"i": {"$lt": 1000}} assert_soon(lambda: retry_until_ok(db_main.find(query).count) > 0) # Only first document should exist in MongoDB self.assertEqual(db_main.find(query).count(), 1) self.assertEqual(db_main.find_one(query)["i"], 0) # Same should hold for the doc manager docman_docs = [d for d in self.opman1.doc_managers[0]._search() if d["i"] < 1000] self.assertEqual(len(docman_docs), 1) self.assertEqual(docman_docs[0]["i"], 0) # Wait for previous rollback to complete def rollback_done(): secondary1_count = retry_until_ok(db_secondary1.count) secondary2_count = retry_until_ok(db_secondary2.count) return (1, 1) == (secondary1_count, secondary2_count) assert_soon(rollback_done, "rollback never replicated to one or more secondaries") ############################## # Case 2: Primaries on both shards go down kill_mongo_proc(self.shard1_prim_p, destroy=False) kill_mongo_proc(self.shard2_prim_p, destroy=False) # Wait for the secondaries to be promoted shard1_secondary_admin = self.shard1_secondary_conn["admin"] shard2_secondary_admin = self.shard2_secondary_conn["admin"] assert_soon( lambda: shard1_secondary_admin.command("isMaster")["ismaster"]) assert_soon( lambda: shard2_secondary_admin.command("isMaster")["ismaster"]) # Insert another document on each shard. These will be rolled back later retry_until_ok(db_main.insert, {"i": 1}) self.assertEqual(db_secondary1.count(), 2) retry_until_ok(db_main.insert, {"i": 1001}) self.assertEqual(db_secondary2.count(), 2) # Wait for replication on the doc manager c = lambda: len(self.opman1.doc_managers[0]._search()) == 4 assert_soon(c, "not all writes were replicated to doc manager") # Kill the new primaries kill_mongo_proc(self.shard1_scnd_p, destroy=False) kill_mongo_proc(self.shard2_scnd_p, destroy=False) # Start the servers back up... # Shard 1 restart_mongo_proc(self.shard1_prim_p) c = lambda: self.shard1_conn['admin'].command("isMaster")["ismaster"] assert_soon(lambda: retry_until_ok(c)) restart_mongo_proc(self.shard1_scnd_p) secondary_admin = self.shard1_secondary_conn["admin"] c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2 assert_soon(c) # Shard 2 restart_mongo_proc(self.shard2_prim_p) c = lambda: self.shard2_conn['admin'].command("isMaster")["ismaster"] assert_soon(lambda: retry_until_ok(c)) restart_mongo_proc(self.shard2_scnd_p) secondary_admin = self.shard2_secondary_conn["admin"] c = lambda: secondary_admin.command("replSetGetStatus")["myState"] == 2 assert_soon(c) # Wait for the shards to come online assert_soon(lambda: retry_until_ok(db_main.find(query).count) > 0) query2 = {"i": {"$gte": 1000}} assert_soon(lambda: retry_until_ok(db_main.find(query2).count) > 0) # Only first documents should exist in MongoDB self.assertEqual(db_main.find(query).count(), 1) self.assertEqual(db_main.find_one(query)["i"], 0) self.assertEqual(db_main.find(query2).count(), 1) self.assertEqual(db_main.find_one(query2)["i"], 1000) # Same should hold for the doc manager i_values = [d["i"] for d in self.opman1.doc_managers[0]._search()] self.assertEqual(len(i_values), 2) self.assertIn(0, i_values) self.assertIn(1000, i_values)
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') test_oplog, primary_conn, mongos, solr = self.get_new_oplog() if not start_cluster(): self.fail('Cluster could not be started successfully!') solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') mongos['test']['test'].remove({}) mongos['test']['test'].insert( {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}, safe=True ) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000001')} #try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['test']['test'].insert({ '_id': ObjectId('4ff74db3f646462b38000002'), 'name': 'paul'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert doc failed too many times') time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): time.sleep(1) kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000002')} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert(len(results) == 1) self.assertEqual(results[0]['name'], 'paulie') self.assertTrue(results[0]['_ts'] <= bson_ts_to_long(cutoff_ts))
def tearDownClass(cls): """ Kills cluster instance """ kill_mongo_proc(cls.standalone_port) kill_replica_set('test-mongo')
def test_many_targets(self): """Test with several replication targets""" # OplogThread has multiple doc managers doc_managers = [DocManager(), DocManager(), DocManager()] self.opman.doc_managers = doc_managers self.opman.start() # Insert a document into each namespace self.main_conn["test"]["mc"].insert({"i": 0}) self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) # Make sure the insert is replicated secondary = self.secondary_conn self.assertTrue(wait_for(lambda: secondary["test"]["mc"].count() == 1), "first write didn't replicate to secondary") # Kill the primary kill_mongo_proc("localhost", PORTS_ONE["PRIMARY"]) # Wait for the secondary to be promoted while not secondary["admin"].command("isMaster")["ismaster"]: time.sleep(1) # Insert more documents. This will be rolled back later # Some of these documents will be manually removed from # certain doc managers, to emulate the effect of certain # target systems being ahead/behind others secondary_ids = [] for i in range(1, 10): secondary_ids.append( retry_until_ok(self.main_conn["test"]["mc"].insert, {"i": i})) self.assertEqual(self.secondary_conn["test"]["mc"].count(), 10) # Wait for replication to the doc managers def docmans_done(): for dm in self.opman.doc_managers: if len(dm._search()) != 10: return False return True self.assertTrue(wait_for(docmans_done), "not all writes were replicated to doc managers") # Remove some documents from the doc managers to simulate # uneven replication for id in secondary_ids[8:]: self.opman.doc_managers[1].remove({"_id": id}) for id in secondary_ids[2:]: self.opman.doc_managers[2].remove({"_id": id}) # Kill the new primary kill_mongo_proc("localhost", PORTS_ONE["SECONDARY"]) # Start both servers back up start_mongo_proc(port=PORTS_ONE['PRIMARY'], repl_set_name="demo-repl", data="/replset1a", log="/replset1a.log", key_file=None) primary_admin = self.primary_conn["admin"] while not primary_admin.command("isMaster")["ismaster"]: time.sleep(1) start_mongo_proc(port=PORTS_ONE['SECONDARY'], repl_set_name="demo-repl", data="/replset1b", log="/replset1b.log", key_file=None) while secondary["admin"].command("replSetGetStatus")["myState"] != 2: time.sleep(1) while retry_until_ok( self.primary_conn["test"]["mc"].find().count) == 0: time.sleep(1) # Only first document should exist in MongoDB self.assertEqual(self.primary_conn["test"]["mc"].count(), 1) self.assertEqual(self.primary_conn["test"]["mc"].find_one()["i"], 0) # Same case should hold for the doc managers for dm in self.opman.doc_managers: self.assertEqual(len(dm._search()), 1) self.assertEqual(dm._search()[0]["i"], 0) self.opman.join()