def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, search_ts = self.get_oplog_thread() #with documents primary_conn['test']['test'].insert({'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() doc_manager = test_oplog.doc_managers[0] doc_manager.commit() solr_results = doc_manager._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts) self.assertEqual(solr_doc['name'], 'paulie') self.assertEqual(solr_doc['ns'], 'test.test') # test multiple targets doc_managers = [DocManager(), DocManager(), DocManager()] test_oplog.doc_managers = doc_managers primary_conn["test"]["test"].remove() for i in range(1000): primary_conn["test"]["test"].insert({"i": i}) test_oplog.dump_collection() for dm in doc_managers: self.assertEqual(len(dm._search()), 1000)
def test_many_targets(self): """Test that one OplogThread is capable of replicating to more than one target. """ opman, primary_conn, oplog_coll = self.get_oplog_thread() doc_managers = [DocManager(), DocManager(), DocManager()] opman.doc_managers = doc_managers # start replicating opman.start() primary_conn["test"]["test"].insert({ "name": "kermit", "color": "green" }) primary_conn["test"]["test"].insert({ "name": "elmo", "color": "firetruck red" }) self.assertTrue( wait_for(lambda: sum(len(d._search()) for d in doc_managers) == 6), "OplogThread should be able to replicate to multiple targets") primary_conn["test"]["test"].remove({"name": "elmo"}) self.assertTrue( wait_for(lambda: sum(len(d._search()) for d in doc_managers) == 3), "OplogThread should be able to replicate to multiple targets") for d in doc_managers: self.assertEqual(d._search()[0]["name"], "kermit") # cleanup opman.join()
def test_many_targets(self): """Test that one OplogThread is capable of replicating to more than one target. """ doc_managers = [DocManager(), DocManager(), DocManager()] self.opman.doc_managers = doc_managers # start replicating self.opman.start() self.primary_conn["test"]["test"].insert_one({ "name": "kermit", "color": "green" }) self.primary_conn["test"]["test"].insert_one({ "name": "elmo", "color": "firetruck red" }) assert_soon( lambda: sum(len(d._search()) for d in doc_managers) == 6, "OplogThread should be able to replicate to multiple targets") self.primary_conn["test"]["test"].delete_one({"name": "elmo"}) assert_soon( lambda: sum(len(d._search()) for d in doc_managers) == 3, "OplogThread should be able to replicate to multiple targets") for d in doc_managers: self.assertEqual(d._search()[0]["name"], "kermit")
def test_start_with_auth(self): dm = DocManager() connector = Connector(mongo_address=self.cluster.uri, doc_managers=[dm], auth_username=db_user, auth_key=db_password) connector.start() # Insert some documents into the sharded cluster. These # should go to the DocManager, and the connector should not # have an auth failure. self.cluster.client().test.test.insert_one({'auth_failure': False}) assert_soon(lambda: len(dm._search()) > 0) connector.join()
def setUp(self): # Create a new oplog progress file try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() # Start a replica set self.repl_set = ReplicaSet().start() # Connection to the replica set as a whole self.main_conn = self.repl_set.client() # Connection to the primary specifically self.primary_conn = self.repl_set.primary.client() # Connection to the secondary specifically self.secondary_conn = self.repl_set.secondary.client( read_preference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread(primary_client=self.main_conn, doc_managers=(doc_manager, ), oplog_progress_dict=oplog_progress, ns_set=["test.mc"])
def setUp(self): self.repl_set = ReplicaSet().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local['oplog.rs'] self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict())
def get_new_oplog(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ is_sharded = True primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]: mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) is_sharded = False oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test'] doc_manager = DocManager() oplog = OplogThread(primary_conn=primary_conn, main_address=mongos_addr, oplog_coll=oplog_coll, is_sharded=is_sharded, doc_manager=doc_manager, oplog_progress_dict=LockingDict(), namespace_set=namespace_set, auth_key=cls.AUTH_KEY, auth_username=AUTH_USERNAME, repl_set="demo-repl") return (oplog, primary_conn, oplog.main_connection, oplog_coll)
def get_oplog_thread(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection. This function clears the oplog. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]) mongos = Connection(mongos_addr) mongos['alpha']['foo'].drop() oplog_coll = primary_conn['local']['oplog.rs'] oplog_coll.drop() # reset the oplog primary_conn['local'].create_collection('oplog.rs', capped=True, size=1000000) namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, True, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, mongos)
def test_skipped_oplog_entry_updates_checkpoint(self): repl_set = ReplicaSetSingle().start() conn = repl_set.client() opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig(namespace_set=["test.test"]), ) opman.start() # Insert a document into an included collection conn["test"]["test"].insert_one({"test": 1}) last_ts = opman.get_last_oplog_timestamp() assert_soon( lambda: last_ts == opman.checkpoint, "OplogThread never updated checkpoint to non-skipped " "entry.", ) self.assertEqual(len(opman.doc_managers[0]._search()), 1) # Make sure that the oplog thread updates its checkpoint on every # oplog entry. conn["test"]["ignored"].insert_one({"test": 1}) last_ts = opman.get_last_oplog_timestamp() assert_soon( lambda: last_ts == opman.checkpoint, "OplogThread never updated checkpoint to skipped entry.", ) opman.join() conn.close() repl_set.stop()
def get_oplog_thread(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function clears the oplog """ is_sharded = True primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) primary_conn['test']['test'].drop() mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]: mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) is_sharded = False oplog_coll = primary_conn['local']['oplog.rs'] oplog_coll.drop() # reset the oplog primary_conn['local'].create_collection('oplog.rs', capped=True, size=1000000) namespace_set = ['test.test'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, is_sharded, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME, repl_set="demo-repl") return(oplog, primary_conn, oplog_coll)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None): self.namespace_config = NamespaceConfig(namespace_set=include_ns, ex_namespace_set=exclude_ns, namespace_options=dest_mapping) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), namespace_config=self.namespace_config)
def test_start_with_auth(self): dm = DocManager() connector = Connector( mongo_address=self.cluster.uri, doc_managers=[dm], auth_username=db_user, auth_key=db_password ) connector.start() # Insert some documents into the sharded cluster. These # should go to the DocManager, and the connector should not # have an auth failure. self.cluster.client().test.test.insert({'auth_failure': False}) assert_soon(lambda: len(dm._search()) > 0) connector.join()
def setUp(self): self.namespace_config = NamespaceConfig() self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), namespace_config=self.namespace_config, )
def test_many_targets(self): """Test that one OplogThread is capable of replicating to more than one target. """ self.reset_opman(["includedb1.*"], [], {}) doc_managers = [DocManager(), DocManager(), DocManager()] self.opman.doc_managers = doc_managers # start replicating self.opman.start() self.primary_conn["includedb1"]["includecol1"].insert_one({ "name": "kermit", "color": "green" }) self.primary_conn["includedb1"]["includecol2"].insert_one({ "name": "elmo", "color": "firetruck red" }) self.primary_conn["excludedb2"]["excludecol1"].insert_one({ "name": "panda", "color": "white and black" }) assert_soon( lambda: sum(len(d._search()) for d in doc_managers) == 6, "OplogThread should be able to replicate to multiple targets", ) self.primary_conn["includedb1"]["includecol2"].delete_one( {"name": "elmo"}) assert_soon( lambda: sum(len(d._search()) for d in doc_managers) == 3, "OplogThread should be able to replicate to multiple targets", ) for d in doc_managers: self.assertEqual(d._search()[0]["name"], "kermit")
def test_fields_and_exclude(self): fields = ['a', 'b', 'c', '_id'] exclude_fields = ['x', 'y', 'z'] # Test setting both to None in constructor opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=None, exclude_fields=None) self._check_fields(opman, [], [], None) opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=None, exclude_fields=exclude_fields) self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) # Test setting fields when exclude_fields is set self.assertRaises(errors.InvalidConfiguration, setattr, opman, "fields", fields) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "fields", None) opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=None, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "exclude_fields", exclude_fields) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "exclude_fields", None) self.assertRaises(errors.InvalidConfiguration, OplogThread, self.primary_conn, (DocManager(), ), LockingDict(), self.dest_mapping_stru, fields=fields, exclude_fields=exclude_fields)
def setUp(self): if db_user and db_password: auth_args = dict(auth_username=db_user, auth_key=db_password) else: auth_args = {} self.cluster = ShardedClusterSingle().start() self.dm = DocManager() self.connector = Connector(mongo_address=self.cluster.uri, doc_managers=[self.dm], **auth_args) self.connector.start()
def setUp(self): self.repl_set = ReplicaSetSingle().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local['oplog.rs'] self.dest_mapping_stru = DestMapping([], [], {}) self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, )
def setUp(self): self.repl_set = ReplicaSetSingle().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local["oplog.rs"] self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig( namespace_options={"test.*": True, "gridfs.*": {"gridfs": True}} ), )
def test_dump_collection(self): """Test the dump_collection method Cases: 1. empty oplog 2. non-empty oplog, with gridfs collections 3. non-empty oplog, specified a namespace-set, none of the oplog entries are for collections in the namespace-set """ # Test with empty oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] last_ts = self.opman.dump_collection() self.assertEqual(last_ts, None) # Test with non-empty oplog with gridfs collections self.opman.oplog = self.primary_conn["local"]["oplog.rs"] # Insert 10 gridfs files for i in range(10): fs = gridfs.GridFS(self.primary_conn["gridfs"], collection="test" + str(i)) fs.put(b"hello world") # Insert 1000 documents for i in range(1000): self.primary_conn["test"]["test"].insert_one({ "i": i + 500 }) last_ts = self.opman.get_last_oplog_timestamp() self.assertEqual(last_ts, self.opman.dump_collection()) self.assertEqual(len(self.opman.doc_managers[0]._search()), 1010) # Case 3 # 1MB oplog so that we can rollover quickly repl_set = ReplicaSetSingle(oplogSize=1).start() conn = repl_set.client() opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig(namespace_set=["test.test"]), ) # Insert a document into an included collection conn["test"]["test"].insert_one({"test": 1}) # Cause the oplog to rollover on a non-included collection while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}): conn["test"]["ignored"].insert_many( [{"test": "1" * 1024} for _ in range(1024)]) last_ts = opman.get_last_oplog_timestamp() self.assertEqual(last_ts, opman.dump_collection()) self.assertEqual(len(opman.doc_managers[0]._search()), 1) conn.close() repl_set.stop()
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, search_ts = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr # with documents primary_conn["test"]["test"].insert({"name": "paulie"}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc["_ts"]), search_ts) self.assertEqual(solr_doc["name"], "paulie") self.assertEqual(solr_doc["ns"], "test.test")
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, search_ts = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr #with documents primary_conn['test']['test'].insert({'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() self.assertEqual(len(solr_results), 1) solr_doc = solr_results[0] self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts) self.assertEqual(solr_doc['name'], 'paulie') self.assertEqual(solr_doc['ns'], 'test.test')
def test_dump_collection(self): """Test dump_collection in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, search_ts, solr, mongos = self.get_oplog_thread() solr = DocManager() test_oplog.doc_manager = solr # with documents safe_mongo_op(mongos['alpha']['foo'].insert, {'name': 'paulie'}) search_ts = test_oplog.get_last_oplog_timestamp() test_oplog.dump_collection() test_oplog.doc_manager.commit() solr_results = solr._search() assert (len(solr_results) == 1) solr_doc = solr_results[0] assert (long_to_bson_ts(solr_doc['_ts']) == search_ts) assert (solr_doc['name'] == 'paulie') assert (solr_doc['ns'] == 'alpha.foo')
def setUpClass(cls): if db_user and db_password: auth_args = dict(auth_username=db_user, auth_key=db_password) else: auth_args = {} cls.cluster = ShardedClusterSingle().start() cls.main_uri = cls.cluster.uri + '/?readPreference=primaryPreferred' cls.dm = DocManager() cls.connector = Connector(mongo_address=cls.main_uri, doc_managers=[cls.dm], **auth_args) cls.connector.start() assert_soon(lambda: len(cls.connector.shard_set) == 2, message='connector failed to find both shards!')
def test_dump_collection(self): """Test the dump_collection method Cases: 1. empty oplog 2. non-empty oplog 3. non-empty oplog, specified a namespace-set, none of the oplog entries are for collections in the namespace-set """ # Test with empty oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] last_ts = self.opman.dump_collection() self.assertEqual(last_ts, None) # Test with non-empty oplog self.opman.oplog = self.primary_conn["local"]["oplog.rs"] for i in range(1000): self.primary_conn["test"]["test"].insert_one({ "i": i + 500 }) last_ts = self.opman.get_last_oplog_timestamp() self.assertEqual(last_ts, self.opman.dump_collection()) self.assertEqual(len(self.opman.doc_managers[0]._search()), 1000) # Case 3 # 1MB oplog so that we can rollover quickly repl_set = ReplicaSetSingle(oplogSize=1).start() conn = repl_set.client() dest_mapping_stru = DestMapping(["test.test"], [], {}) opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), dest_mapping_stru=dest_mapping_stru, ns_set=set(["test.test"]) ) # Insert a document into a ns_set collection conn["test"]["test"].insert_one({"test": 1}) # Cause the oplog to rollover on a non-ns_set collection while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}): conn["test"]["ignored"].insert_many( [{"test": "1" * 1024} for _ in range(1024)]) last_ts = opman.get_last_oplog_timestamp() self.assertEqual(last_ts, opman.dump_collection()) self.assertEqual(len(opman.doc_managers[0]._search()), 1) conn.close() repl_set.stop()
def setUp(self): _, _, self.primary_p = start_replica_set('test-oplog-manager') self.primary_conn = pymongo.MongoClient(mongo_host, self.primary_p) self.oplog_coll = self.primary_conn.local['oplog.rs'] self.opman = OplogThread(primary_conn=self.primary_conn, main_address='%s:%d' % (mongo_host, self.primary_p), oplog_coll=self.oplog_coll, is_sharded=False, doc_manager=DocManager(), oplog_progress_dict=LockingDict(), namespace_set=None, auth_key=None, auth_username=None, repl_set='test-oplog-manager')
def test_init_cursor(self): """Test init_cursor in oplog_manager. Assertion failure if it doesn't pass """ test_oplog, primary_conn, search_ts = self.get_oplog_thread() test_oplog.checkpoint = None # needed for these tests # initial tests with no config file and empty oplog self.assertEqual(test_oplog.init_cursor(), None) # no config, single oplog entry primary_conn["test"]["test"].insert({"name": "paulie"}) search_ts = test_oplog.get_last_oplog_timestamp() cursor = test_oplog.init_cursor() self.assertEqual(cursor.count(), 1) self.assertEqual(test_oplog.checkpoint, search_ts) # with config file, assert that size != 0 os.system("touch temp_config.txt") cursor = test_oplog.init_cursor() oplog_dict = test_oplog.oplog_progress.get_dict() self.assertEqual(cursor.count(), 1) self.assertTrue(str(test_oplog.oplog) in oplog_dict) self.assertTrue(oplog_dict[str(test_oplog.oplog)] == test_oplog.checkpoint) os.system("rm temp_config.txt") # test init_cursor when OplogThread created with/without no-dump option # insert some documents (will need to be dumped) primary_conn["test"]["test"].remove() primary_conn["test"]["test"].insert(({"_id": i} for i in range(100))) # test no-dump option docman = DocManager() docman._delete() test_oplog.doc_manager = docman test_oplog.collection_dump = False test_oplog.oplog_progress = LockingDict() # init_cursor has the side-effect of causing a collection dump test_oplog.init_cursor() self.assertEqual(len(docman._search()), 0) # test w/o no-dump option docman._delete() test_oplog.collection_dump = True test_oplog.oplog_progress = LockingDict() test_oplog.init_cursor() self.assertEqual(len(docman._search()), 100)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None): if include_ns is None: include_ns = [] if exclude_ns is None: exclude_ns = [] if dest_mapping is None: dest_mapping = {} # include_ns must not exist together with exclude_ns # dest_mapping must exist together with include_ns # those checks have been tested in test_config.py so we skip that here. self.dest_mapping_stru = DestMapping(include_ns, exclude_ns, dest_mapping) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, ns_set=include_ns, ex_ns_set=exclude_ns)
def get_new_oplog(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]) oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos, oplog_coll, True, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, oplog.main_connection)
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set _, self.secondary_p, self.primary_p = start_replica_set('rollbacks') # Connection to the replica set as a whole self.main_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p), replicaSet='rollbacks') # Connection to the primary specifically self.primary_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p)) # Connection to the secondary specifically self.secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.secondary_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address='%s:%d' % (mongo_host, self.primary_p), oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="rollbacks" )
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set start_cluster(sharded=False, use_mongos=False) # Connection to the replica set as a whole self.main_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"], replicaSet="demo-repl") # Connection to the primary specifically self.primary_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"]) # Connection to the secondary specifically self.secondary_conn = Connection( "localhost:%s" % PORTS_ONE["SECONDARY"], read_preference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address="localhost:%s" % PORTS_ONE["PRIMARY"], oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="demo-repl")
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') test_oplog, primary_conn, mongos, solr = self.get_new_oplog() if not start_cluster(): self.fail('Cluster could not be started successfully!') solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') mongos['test']['test'].remove({}) mongos['test']['test'].insert( {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}, safe=True ) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000001')} #try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['test']['test'].insert({ '_id': ObjectId('4ff74db3f646462b38000002'), 'name': 'paul'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert doc failed too many times') time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): time.sleep(1) kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000002')} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert(len(results) == 1) self.assertEqual(results[0]['name'], 'paulie') self.assertTrue(results[0]['_ts'] <= bson_ts_to_long(cutoff_ts))
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing primary, inserting another doc, killing the new primary, and then restarting both servers. """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) if not start_cluster(sharded=True): self.fail("Shards cannot be added to mongos") test_oplog, primary_conn, solr, mongos = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*:*') safe_mongo_op(mongos['alpha']['foo'].remove, {}) safe_mongo_op(mongos['alpha']['foo'].insert, {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'alpha.foo', '_id': ObjectId('4ff74db3f646462b38000001')} # try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'}) break except OperationFailure: time.sleep(1) count += 1 if count > 60: self.fail('Insert failed too many times in rollback') continue kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) # wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) # wait for secondary to be established admin_db = new_primary_conn['admin'] while admin_db.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['alpha']['foo'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'alpha.foo', '_id': obj2} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() self.assertEqual(len(results), 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def setUp(self): self.dest_mapping_stru = DestMapping([], [], {}) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru)
def test_exclude_fields_constructor(self): # Test with the "_id" field in exclude_fields exclude_fields = ["_id", "title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) exclude_fields.remove('_id') self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) extra_fields = exclude_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered) # Test without "_id" field included in exclude_fields exclude_fields = ["title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) extra_fields = extra_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual({'extra1': 1, 'extra2': 1}, filtered) # Test with only "_id" field in exclude_fields exclude_fields = ["_id"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) self._check_fields(opman, [], [], None) extra_fields = exclude_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered) # Test with nothing set for exclude_fields opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=None) self._check_fields(opman, [], [], None) extra_fields = ['_id', 'extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_constructor(self): # Test with "_id" field in constructor fields = ["_id", "title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test without "_id" field in constructor fields = ["title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) fields.append('_id') self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test with only "_id" field fields = ["_id"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual({'_id': 1}, filtered) # Test with no fields set opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru) self._check_fields(opman, [], [], None) extra_fields = ['_id', 'extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered)