def test_skipped_oplog_entry_updates_checkpoint(self): repl_set = ReplicaSetSingle().start() conn = repl_set.client() opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig(namespace_set=["test.test"]), ) opman.start() # Insert a document into an included collection conn["test"]["test"].insert_one({"test": 1}) last_ts = opman.get_last_oplog_timestamp() assert_soon( lambda: last_ts == opman.checkpoint, "OplogThread never updated checkpoint to non-skipped " "entry.", ) self.assertEqual(len(opman.doc_managers[0]._search()), 1) # Make sure that the oplog thread updates its checkpoint on every # oplog entry. conn["test"]["ignored"].insert_one({"test": 1}) last_ts = opman.get_last_oplog_timestamp() assert_soon( lambda: last_ts == opman.checkpoint, "OplogThread never updated checkpoint to skipped entry.", ) opman.join() conn.close() repl_set.stop()
def setUp(self): self.repl_set = ReplicaSet().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local['oplog.rs'] self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict())
def setUp(self): # Create a new oplog progress file try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() # Start a replica set self.repl_set = ReplicaSet().start() # Connection to the replica set as a whole self.main_conn = self.repl_set.client() # Connection to the primary specifically self.primary_conn = self.repl_set.primary.client() # Connection to the secondary specifically self.secondary_conn = self.repl_set.secondary.client( read_preference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread(primary_client=self.main_conn, doc_managers=(doc_manager, ), oplog_progress_dict=oplog_progress, ns_set=["test.mc"])
def get_oplog_thread(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function clears the oplog """ is_sharded = True primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) primary_conn['test']['test'].drop() mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]: mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) is_sharded = False oplog_coll = primary_conn['local']['oplog.rs'] oplog_coll.drop() # reset the oplog primary_conn['local'].create_collection('oplog.rs', capped=True, size=1000000) namespace_set = ['test.test'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, is_sharded, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME, repl_set="demo-repl") return(oplog, primary_conn, oplog_coll)
def get_oplog_thread(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection. This function clears the oplog. """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]) mongos = Connection(mongos_addr) mongos['alpha']['foo'].drop() oplog_coll = primary_conn['local']['oplog.rs'] oplog_coll.drop() # reset the oplog primary_conn['local'].create_collection('oplog.rs', capped=True, size=1000000) namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, True, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, mongos)
def get_new_oplog(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ is_sharded = True primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]: mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN']) is_sharded = False oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test'] doc_manager = DocManager() oplog = OplogThread(primary_conn=primary_conn, main_address=mongos_addr, oplog_coll=oplog_coll, is_sharded=is_sharded, doc_manager=doc_manager, oplog_progress_dict=LockingDict(), namespace_set=namespace_set, auth_key=cls.AUTH_KEY, auth_username=AUTH_USERNAME, repl_set="demo-repl") return (oplog, primary_conn, oplog.main_connection, oplog_coll)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None): self.namespace_config = NamespaceConfig(namespace_set=include_ns, ex_namespace_set=exclude_ns, namespace_options=dest_mapping) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), namespace_config=self.namespace_config)
def setUp(self): self.namespace_config = NamespaceConfig() self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), namespace_config=self.namespace_config, )
def initOplogThread(self, namespace_set=[], dest_mapping={}): self.docman = CommandLoggerDocManager() self.docman.command_helper = CommandHelper(namespace_set, dest_mapping) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(self.docman, ), oplog_progress_dict=self.oplog_progress, namespace_set=namespace_set, dest_mapping=dest_mapping, collection_dump=False) self.opman.start()
def test_fields_and_exclude(self): fields = ['a', 'b', 'c', '_id'] exclude_fields = ['x', 'y', 'z'] # Test setting both to None in constructor opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=None, exclude_fields=None) self._check_fields(opman, [], [], None) opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=None, exclude_fields=exclude_fields) self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) # Test setting fields when exclude_fields is set self.assertRaises(errors.InvalidConfiguration, setattr, opman, "fields", fields) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "fields", None) opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=None, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "exclude_fields", exclude_fields) self.assertRaises(errors.InvalidConfiguration, setattr, opman, "exclude_fields", None) self.assertRaises(errors.InvalidConfiguration, OplogThread, self.primary_conn, (DocManager(), ), LockingDict(), self.dest_mapping_stru, fields=fields, exclude_fields=exclude_fields)
def setUp(self): self.repl_set = ReplicaSetSingle().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local['oplog.rs'] self.dest_mapping_stru = DestMapping([], [], {}) self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, )
def initOplogThread(self, namespace_set=None): self.docman = CommandLoggerDocManager() namespace_config = NamespaceConfig(namespace_set=namespace_set) self.docman.command_helper = CommandHelper(namespace_config) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(self.docman, ), oplog_progress_dict=self.oplog_progress, namespace_config=namespace_config, collection_dump=False) self.opman.start()
def setUp(self): self.repl_set = ReplicaSetSingle().start() self.primary_conn = self.repl_set.client() self.oplog_coll = self.primary_conn.local["oplog.rs"] self.opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig( namespace_options={"test.*": True, "gridfs.*": {"gridfs": True}} ), )
def test_dump_collection(self): """Test the dump_collection method Cases: 1. empty oplog 2. non-empty oplog, with gridfs collections 3. non-empty oplog, specified a namespace-set, none of the oplog entries are for collections in the namespace-set """ # Test with empty oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] last_ts = self.opman.dump_collection() self.assertEqual(last_ts, None) # Test with non-empty oplog with gridfs collections self.opman.oplog = self.primary_conn["local"]["oplog.rs"] # Insert 10 gridfs files for i in range(10): fs = gridfs.GridFS(self.primary_conn["gridfs"], collection="test" + str(i)) fs.put(b"hello world") # Insert 1000 documents for i in range(1000): self.primary_conn["test"]["test"].insert_one({ "i": i + 500 }) last_ts = self.opman.get_last_oplog_timestamp() self.assertEqual(last_ts, self.opman.dump_collection()) self.assertEqual(len(self.opman.doc_managers[0]._search()), 1010) # Case 3 # 1MB oplog so that we can rollover quickly repl_set = ReplicaSetSingle(oplogSize=1).start() conn = repl_set.client() opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), namespace_config=NamespaceConfig(namespace_set=["test.test"]), ) # Insert a document into an included collection conn["test"]["test"].insert_one({"test": 1}) # Cause the oplog to rollover on a non-included collection while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}): conn["test"]["ignored"].insert_many( [{"test": "1" * 1024} for _ in range(1024)]) last_ts = opman.get_last_oplog_timestamp() self.assertEqual(last_ts, opman.dump_collection()) self.assertEqual(len(opman.doc_managers[0]._search()), 1) conn.close() repl_set.stop()
def test_dump_collection(self): """Test the dump_collection method Cases: 1. empty oplog 2. non-empty oplog 3. non-empty oplog, specified a namespace-set, none of the oplog entries are for collections in the namespace-set """ # Test with empty oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] last_ts = self.opman.dump_collection() self.assertEqual(last_ts, None) # Test with non-empty oplog self.opman.oplog = self.primary_conn["local"]["oplog.rs"] for i in range(1000): self.primary_conn["test"]["test"].insert_one({ "i": i + 500 }) last_ts = self.opman.get_last_oplog_timestamp() self.assertEqual(last_ts, self.opman.dump_collection()) self.assertEqual(len(self.opman.doc_managers[0]._search()), 1000) # Case 3 # 1MB oplog so that we can rollover quickly repl_set = ReplicaSetSingle(oplogSize=1).start() conn = repl_set.client() dest_mapping_stru = DestMapping(["test.test"], [], {}) opman = OplogThread( primary_client=conn, doc_managers=(DocManager(),), oplog_progress_dict=LockingDict(), dest_mapping_stru=dest_mapping_stru, ns_set=set(["test.test"]) ) # Insert a document into a ns_set collection conn["test"]["test"].insert_one({"test": 1}) # Cause the oplog to rollover on a non-ns_set collection while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}): conn["test"]["ignored"].insert_many( [{"test": "1" * 1024} for _ in range(1024)]) last_ts = opman.get_last_oplog_timestamp() self.assertEqual(last_ts, opman.dump_collection()) self.assertEqual(len(opman.doc_managers[0]._search()), 1) conn.close() repl_set.stop()
def setUp(self): _, _, self.primary_p = start_replica_set('test-oplog-manager') self.primary_conn = pymongo.MongoClient(mongo_host, self.primary_p) self.oplog_coll = self.primary_conn.local['oplog.rs'] self.opman = OplogThread(primary_conn=self.primary_conn, main_address='%s:%d' % (mongo_host, self.primary_p), oplog_coll=self.oplog_coll, is_sharded=False, doc_manager=DocManager(), oplog_progress_dict=LockingDict(), namespace_set=None, auth_key=None, auth_username=None, repl_set='test-oplog-manager')
def initOplogThread(self, namespace_set=[], ex_namespace_set=[], dest_mapping={}): self.docman = CommandLoggerDocManager() # Replace the origin dest_mapping self.dest_mapping_stru = DestMapping(namespace_set, ex_namespace_set, dest_mapping) self.docman.command_helper = CommandHelper(self.dest_mapping_stru) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(self.docman, ), oplog_progress_dict=self.oplog_progress, dest_mapping_stru=self.dest_mapping_stru, ns_set=namespace_set, ex_ns_set=ex_namespace_set, collection_dump=False) self.opman.start()
def get_new_oplog(cls): """ Set up connection with mongo. Returns oplog, the connection and oplog collection This function does not clear the oplog """ primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"])) if primary_conn['admin'].command("isMaster")['ismaster'] is False: primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"])) mongos = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]) oplog_coll = primary_conn['local']['oplog.rs'] namespace_set = ['test.test', 'alpha.foo'] doc_manager = DocManager() oplog = OplogThread(primary_conn, mongos, oplog_coll, True, doc_manager, LockingDict(), namespace_set, cls.AUTH_KEY, AUTH_USERNAME) return (oplog, primary_conn, oplog_coll, oplog.main_connection)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None): if include_ns is None: include_ns = [] if exclude_ns is None: exclude_ns = [] if dest_mapping is None: dest_mapping = {} # include_ns must not exist together with exclude_ns # dest_mapping must exist together with include_ns # those checks have been tested in test_config.py so we skip that here. self.dest_mapping_stru = DestMapping(include_ns, exclude_ns, dest_mapping) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, ns_set=include_ns, ex_ns_set=exclude_ns)
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set _, self.secondary_p, self.primary_p = start_replica_set('rollbacks') # Connection to the replica set as a whole self.main_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p), replicaSet='rollbacks') # Connection to the primary specifically self.primary_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p)) # Connection to the secondary specifically self.secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.secondary_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address='%s:%d' % (mongo_host, self.primary_p), oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="rollbacks" )
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set start_cluster(sharded=False, use_mongos=False) # Connection to the replica set as a whole self.main_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"], replicaSet="demo-repl") # Connection to the primary specifically self.primary_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"]) # Connection to the secondary specifically self.secondary_conn = Connection( "localhost:%s" % PORTS_ONE["SECONDARY"], read_preference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address="localhost:%s" % PORTS_ONE["PRIMARY"], oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="demo-repl")
def setUp(self): self.dest_mapping_stru = DestMapping([], [], {}) self.opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru)
def test_fields_constructor(self): # Test with "_id" field in constructor fields = ["_id", "title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test without "_id" field in constructor fields = ["title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) fields.append('_id') self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test with only "_id" field fields = ["_id"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, fields=fields) self._check_fields(opman, fields, [], dict((f, 1) for f in fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual({'_id': 1}, filtered) # Test with no fields set opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru) self._check_fields(opman, [], [], None) extra_fields = ['_id', 'extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_exclude_fields_constructor(self): # Test with the "_id" field in exclude_fields exclude_fields = ["_id", "title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) exclude_fields.remove('_id') self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) extra_fields = exclude_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered) # Test without "_id" field included in exclude_fields exclude_fields = ["title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) self._check_fields(opman, [], exclude_fields, dict((f, 0) for f in exclude_fields)) extra_fields = extra_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual({'extra1': 1, 'extra2': 1}, filtered) # Test with only "_id" field in exclude_fields exclude_fields = ["_id"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=exclude_fields) self._check_fields(opman, [], [], None) extra_fields = exclude_fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered) # Test with nothing set for exclude_fields opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), dest_mapping_stru=self.dest_mapping_stru, exclude_fields=None) self._check_fields(opman, [], [], None) extra_fields = ['_id', 'extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ self.main_conn = self.create_authed_client() LOG.always('Source MongoDB version: %s', self.main_conn.admin.command('buildInfo')['version']) for dm in self.doc_managers: name = dm.__class__.__module__ module = sys.modules[name] version = 'unknown' if hasattr(module, '__version__'): version = module.__version__ elif hasattr(module, 'version'): version = module.version LOG.always('Target DocManager: %s version: %s', name, version) self.read_oplog_progress() conn_type = None try: self.main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": # Make sure we are connected to a replica set is_master = self.main_conn.admin.command("isMaster") if "setName" not in is_master: LOG.error( 'No replica set at "%s"! A replica set is required ' 'to run mongo-connector. Shutting down...' % self.address ) return # Establish a connection to the replica set as a whole self.main_conn.close() self.main_conn = self.create_authed_client( replicaSet=is_master['setName']) # non sharded configuration oplog = OplogThread( self.main_conn, self.doc_managers, self.oplog_progress, self.dest_mapping, **self.kwargs) self.shard_set[0] = oplog LOG.info('MongoConnector: Starting connection thread %s' % self.main_conn) oplog.start() while self.can_run: shard_thread = self.shard_set[0] if not (shard_thread.running and shard_thread.is_alive()): LOG.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run: for shard_doc in retry_until_ok(self.main_conn.admin.command, 'listShards')['shards']: shard_id = shard_doc['_id'] if shard_id in self.shard_set: shard_thread = self.shard_set[shard_id] if not (shard_thread.running and shard_thread.is_alive()): LOG.error("MongoConnector: OplogThread " "%s unexpectedly stopped! Shutting " "down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" LOG.exception("MongoConnector: %s", cause) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return shard_conn = self.create_authed_client( hosts, replicaSet=repl_set) oplog = OplogThread( shard_conn, self.doc_managers, self.oplog_progress, self.dest_mapping, mongos_client=self.main_conn, **self.kwargs) self.shard_set[shard_id] = oplog msg = "Starting connection thread" LOG.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() if self.signal is not None: LOG.info("recieved signal %s: shutting down...", self.signal) self.oplog_thread_join() self.write_oplog_progress()
def setUp(self): """ Initialize the cluster: Clean out the databases used by the tests Make connections to mongos, mongods Create and shard test collections Create OplogThreads """ self.cluster = ShardedCluster().start() # Connection to mongos self.mongos_conn = self.cluster.client() # Connections to the shards self.shard1_conn = self.cluster.shards[0].client() self.shard2_conn = self.cluster.shards[1].client() self.shard1_secondary_conn = self.cluster.shards[0].secondary.client( readPreference=ReadPreference.SECONDARY_PREFERRED) self.shard2_secondary_conn = self.cluster.shards[1].secondary.client( readPreference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.mongos_conn["test"]["mcsharded"].drop() # Create and shard the collection test.mcsharded on the "i" field self.mongos_conn["test"]["mcsharded"].ensure_index("i") self.mongos_conn.admin.command("enableSharding", "test") self.mongos_conn.admin.command("shardCollection", "test.mcsharded", key={"i": 1}) # Pre-split the collection so that: # i < 1000 lives on shard1 # i >= 1000 lives on shard2 self.mongos_conn.admin.command( bson.SON([("split", "test.mcsharded"), ("middle", { "i": 1000 })])) # disable the balancer self.mongos_conn.config.settings.update({"_id": "balancer"}, {"$set": { "stopped": True }}, upsert=True) # Move chunks to their proper places try: self.mongos_conn["admin"].command("moveChunk", "test.mcsharded", find={"i": 1}, to='demo-set-0') except pymongo.errors.OperationFailure: pass try: self.mongos_conn["admin"].command("moveChunk", "test.mcsharded", find={"i": 1000}, to='demo-set-1') except pymongo.errors.OperationFailure: pass # Make sure chunks are distributed correctly self.mongos_conn["test"]["mcsharded"].insert({"i": 1}) self.mongos_conn["test"]["mcsharded"].insert({"i": 1000}) def chunks_moved(): doc1 = self.shard1_conn.test.mcsharded.find_one() doc2 = self.shard2_conn.test.mcsharded.find_one() if None in (doc1, doc2): return False return doc1['i'] == 1 and doc2['i'] == 1000 assert_soon(chunks_moved, max_tries=120, message='chunks not moved? doc1=%r, doc2=%r' % (self.shard1_conn.test.mcsharded.find_one(), self.shard2_conn.test.mcsharded.find_one())) self.mongos_conn.test.mcsharded.remove() # create a new oplog progress file try: os.unlink("oplog.timestamp") except OSError: pass open("oplog.timestamp", "w").close() # Oplog threads (oplog manager) for each shard doc_manager = DocManager() oplog_progress = LockingDict() self.opman1 = OplogThread( primary_client=self.shard1_conn, doc_managers=(doc_manager, ), oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], mongos_client=self.mongos_conn) self.opman2 = OplogThread( primary_client=self.shard2_conn, doc_managers=(doc_manager, ), oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], mongos_client=self.mongos_conn)
def setUp(self): """ Initialize the cluster: Clean out the databases used by the tests Make connections to mongos, mongods Create and shard test collections Create OplogThreads """ # Start the cluster with a mongos on port 27217 self.mongos_p = start_cluster() # Connection to mongos mongos_address = '%s:%d' % (mongo_host, self.mongos_p) self.mongos_conn = MongoClient(mongos_address) # Connections to the shards shard1_ports = get_shard(self.mongos_p, 0) shard2_ports = get_shard(self.mongos_p, 1) self.shard1_prim_p = shard1_ports['primary'] self.shard1_scnd_p = shard1_ports['secondaries'][0] self.shard2_prim_p = shard2_ports['primary'] self.shard2_scnd_p = shard2_ports['secondaries'][0] self.shard1_conn = MongoClient('%s:%d' % (mongo_host, self.shard1_prim_p), replicaSet="demo-set-0") self.shard2_conn = MongoClient('%s:%d' % (mongo_host, self.shard2_prim_p), replicaSet="demo-set-1") self.shard1_secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.shard1_scnd_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) self.shard2_secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.shard2_scnd_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) # Wipe any test data self.mongos_conn["test"]["mcsharded"].drop() # Create and shard the collection test.mcsharded on the "i" field self.mongos_conn["test"]["mcsharded"].ensure_index("i") self.mongos_conn.admin.command("enableSharding", "test") self.mongos_conn.admin.command("shardCollection", "test.mcsharded", key={"i": 1}) # Pre-split the collection so that: # i < 1000 lives on shard1 # i >= 1000 lives on shard2 self.mongos_conn.admin.command(bson.SON([ ("split", "test.mcsharded"), ("middle", {"i": 1000}) ])) # disable the balancer self.mongos_conn.config.settings.update( {"_id": "balancer"}, {"$set": {"stopped": True}}, upsert=True ) # Move chunks to their proper places try: self.mongos_conn["admin"].command( "moveChunk", "test.mcsharded", find={"i": 1}, to="demo-set-0" ) except pymongo.errors.OperationFailure: pass # chunk may already be on the correct shard try: self.mongos_conn["admin"].command( "moveChunk", "test.mcsharded", find={"i": 1000}, to="demo-set-1" ) except pymongo.errors.OperationFailure: pass # chunk may already be on the correct shard # Make sure chunks are distributed correctly self.mongos_conn["test"]["mcsharded"].insert({"i": 1}) self.mongos_conn["test"]["mcsharded"].insert({"i": 1000}) def chunks_moved(): doc1 = self.shard1_conn.test.mcsharded.find_one() doc2 = self.shard2_conn.test.mcsharded.find_one() if None in (doc1, doc2): return False return doc1['i'] == 1 and doc2['i'] == 1000 assert_soon(chunks_moved) self.mongos_conn.test.mcsharded.remove() # create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Oplog threads (oplog manager) for each shard doc_manager = DocManager() oplog_progress = LockingDict() self.opman1 = OplogThread( primary_conn=self.shard1_conn, main_address='%s:%d' % (mongo_host, self.mongos_p), oplog_coll=self.shard1_conn["local"]["oplog.rs"], is_sharded=True, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], auth_key=None, auth_username=None ) self.opman2 = OplogThread( primary_conn=self.shard2_conn, main_address='%s:%d' % (mongo_host, self.mongos_p), oplog_coll=self.shard2_conn["local"]["oplog.rs"], is_sharded=True, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], auth_key=None, auth_username=None )
def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ main_conn = MongoClient(self.address) if self.auth_key is not None: main_conn['admin'].authenticate(self.auth_username, self.auth_key) self.read_oplog_progress() conn_type = None try: main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": # Make sure we are connected to a replica set is_master = main_conn.admin.command("isMaster") if not "setName" in is_master: logging.error( 'No replica set at "%s"! A replica set is required ' 'to run mongo-connector. Shutting down...' % self.address) return # Establish a connection to the replica set as a whole main_conn.disconnect() main_conn = MongoClient(self.address, replicaSet=is_master['setName']) if self.auth_key is not None: main_conn.admin.authenticate(self.auth_username, self.auth_key) #non sharded configuration oplog_coll = main_conn['local']['oplog.rs'] oplog = OplogThread(primary_conn=main_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=False, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, repl_set=is_master['setName'], collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error) self.shard_set[0] = oplog logging.info('MongoConnector: Starting connection thread %s' % main_conn) oplog.start() while self.can_run: if not self.shard_set[0].running: logging.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run is True: for shard_doc in main_conn['config']['shards'].find(): shard_id = shard_doc['_id'] if shard_id in self.shard_set: if not self.shard_set[shard_id].running: logging.error("MongoConnector: OplogThread " "%s unexpectedly stopped! Shutting " "down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" logging.error("MongoConnector: %s", cause) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return shard_conn = MongoClient(hosts, replicaSet=repl_set) oplog_coll = shard_conn['local']['oplog.rs'] oplog = OplogThread( primary_conn=shard_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=True, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error) self.shard_set[shard_id] = oplog msg = "Starting connection thread" logging.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() self.oplog_thread_join() self.write_oplog_progress()
def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ main_conn = MongoClient(self.address, tz_aware=self.tz_aware, **self.ssl_kwargs) if self.auth_key is not None: main_conn['admin'].authenticate(self.auth_username, self.auth_key) self.read_oplog_progress() conn_type = None try: main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": # Make sure we are connected to a replica set is_master = main_conn.admin.command("isMaster") if "setName" not in is_master: LOG.error('No replica set at "%s"! A replica set is required ' 'to run mongo-connector. Shutting down...' % self.address) return # Establish a connection to the replica set as a whole main_conn.close() main_conn = MongoClient(self.address, replicaSet=is_master['setName'], tz_aware=self.tz_aware, **self.ssl_kwargs) if self.auth_key is not None: main_conn.admin.authenticate(self.auth_username, self.auth_key) # non sharded configuration oplog = OplogThread(main_conn, self.doc_managers, self.oplog_progress, **self.kwargs) self.shard_set[0] = oplog LOG.info('MongoConnector: Starting connection thread %s' % main_conn) oplog.start() while self.can_run: if not self.shard_set[0].running: LOG.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run is True: for shard_doc in main_conn['config']['shards'].find(): shard_id = shard_doc['_id'] if shard_id in self.shard_set: if not self.shard_set[shard_id].running: LOG.error("MongoConnector: OplogThread " "%s unexpectedly stopped! Shutting " "down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" LOG.exception("MongoConnector: %s", cause) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return shard_conn = MongoClient(hosts, replicaSet=repl_set, tz_aware=self.tz_aware, **self.ssl_kwargs) if self.auth_key is not None: shard_conn['admin'].authenticate( self.auth_username, self.auth_key) oplog = OplogThread(shard_conn, self.doc_managers, self.oplog_progress, **self.kwargs) self.shard_set[shard_id] = oplog msg = "Starting connection thread" LOG.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() self.oplog_thread_join() self.write_oplog_progress()
def test_fields(self): # Test with "_id" field in constructor fields = ["_id", "title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), fields=fields) self.assertEqual(set(fields), opman._fields) self.assertEqual(sorted(fields), sorted(opman.fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test without "_id" field in constructor fields = ["title", "content", "author"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), fields=fields) self.assertEqual(set(fields), opman._fields) self.assertEqual(sorted(fields), sorted(opman.fields)) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in fields), filtered) # Test with only "_id" field fields = ["_id"] opman = OplogThread(primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), fields=fields) self.assertEqual(set(fields), opman._fields) self.assertEqual(fields, opman.fields) extra_fields = fields + ['extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual({'_id': 1}, filtered) # Test with no fields set opman = OplogThread( primary_client=self.primary_conn, doc_managers=(DocManager(), ), oplog_progress_dict=LockingDict(), ) self.assertEqual(set([]), opman._fields) self.assertEqual(None, opman.fields) extra_fields = ['_id', 'extra1', 'extra2'] filtered = opman.filter_oplog_entry({ 'op': 'i', 'o': dict((f, 1) for f in extra_fields) })['o'] self.assertEqual(dict((f, 1) for f in extra_fields), filtered)