def get_new_oplog(self): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ is_sharded = True primary_conn = Connection('localhost', int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection('localhost', int(PORTS_ONE["SECONDARY"])) mongos_addr = "localhost:" + PORTS_ONE["MAIN"] if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]: mongos_addr = "localhost:" + PORTS_ONE["MAIN"] is_sharded = False oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, is_sharded, doc_manager, LockingDict(), namespace_set, AUTH_KEY, AUTH_USERNAME, repl_set="demo-repl") return (oplog, primary_conn, oplog.main_connection, oplog_coll)
def get_oplog_thread(self): """ Set up connection with mongo. Returns oplog, the connection and oplog collection. This function clears the oplog. """ primary_conn = Connection('localhost', int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection('localhost', int(PORTS_ONE["SECONDARY"])) mongos_addr = "localhost:" + PORTS_ONE["MONGOS"] mongos = Connection(mongos_addr) mongos['alpha']['foo'].drop() oplog_coll = primary_conn['local']['oplog.rs'] oplog_coll.drop() # reset the oplog primary_conn['local'].create_collection('oplog.rs', capped=True, size=1000000) namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, True, doc_manager, LockingDict(), namespace_set, AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, mongos)
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, search_ts = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr #with documents primary_conn['test']['test'].insert({'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts) self.assertEqual(solr_doc['name'], 'paulie') self.assertEqual(solr_doc['ns'], 'test.test')
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, oplog_coll = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr #with documents primary_conn['test']['test'].insert({'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts) self.assertEqual(solr_doc['name'], 'paulie') self.assertEqual(solr_doc['ns'], 'test.test') #test_oplog.join() print("PASSED TEST DUMP COLLECTION")
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, oplog_coll, mongos = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr # with documents safe_mongo_op(mongos['alpha']['foo'].insert, {'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() assert (len(solr_results) == 1) solr_doc = solr_results[0] assert (long_to_bson_ts(solr_doc['_ts']) == search_ts) assert (solr_doc['name'] == 'paulie') assert (solr_doc['ns'] == 'alpha.foo') print("PASSED TEST DUMP COLLECTION")
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, oplog_coll = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr # with documents primary_conn["test"]["test"].insert({"name": "paulie"}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc["_ts"]), search_ts) self.assertEqual(solr_doc["name"], "paulie") self.assertEqual(solr_doc["ns"], "test.test") # test_oplog.join() print("PASSED TEST DUMP COLLECTION")
def get_new_oplog(self): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ primary_conn = Connection('localhost', int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection('localhost', int(PORTS_ONE["SECONDARY"])) mongos = "localhost:" + PORTS_ONE["MONGOS"] oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos, oplog_coll, True, doc_manager, LockingDict(), namespace_set, AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, oplog.main_connection)
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') start_cluster() test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') obj1 = ObjectId('4ff74db3f646462b38000001') mongos['test']['test'].remove({}) mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'}, safe=1) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': obj1} #try kill one, try restarting killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: current_conn = mongos['test']['test'] current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1) break except: count += 1 if count > 60: string = 'Call to insert doc failed too many times' logging.error(string) sys.exit(1) time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): print(mongos['test']['test'].find().count()) time.sleep(1) killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY']) startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': obj2} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert(len(results) == 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts)) #test_oplog.join() print("PASSED TEST ROLLBACK")
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') start_cluster() test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') obj1 = ObjectId('4ff74db3f646462b38000001') mongos['test']['test'].remove({}) mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'}, safe=1) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = { 'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': obj1 } #try kill one, try restarting killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: current_conn = mongos['test']['test'] current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1) break except: count += 1 if count > 60: string = 'Call to insert doc failed too many times' logging.error(string) sys.exit(1) time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): print(mongos['test']['test'].find().count()) time.sleep(1) killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY']) startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = { 'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': obj2 } test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert (len(results) == 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts)) #test_oplog.join() print("PASSED TEST ROLLBACK")