def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. no last checkpoint, no collection dump, stuff in oplog 6. last checkpoint exists 7. last checkpoint is behind """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman1.oplog = self.shard1_conn["test"]["emptycollection"] self.opman2.oplog = self.shard2_conn["test"]["emptycollection"] self.opman1.collection_dump = False self.opman2.collection_dump = False self.assertTrue( all(doc['op'] == 'n' for doc in self.opman1.init_cursor()[0])) self.assertEqual(self.opman1.checkpoint, None) self.assertTrue( all(doc['op'] == 'n' for doc in self.opman2.init_cursor()[0])) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman1.collection_dump = self.opman2.collection_dump = True cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman1.checkpoint, None) cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman1.oplog = self.shard1_conn["local"]["oplog.rs"] self.opman2.oplog = self.shard2_conn["local"]["oplog.rs"] oplog_startup_ts = self.opman2.get_last_oplog_timestamp() collection = self.mongos_conn["test"]["mcsharded"] collection.insert({"i": 1}) collection.remove({"i": 1}) time.sleep(3) last_ts1 = self.opman1.get_last_oplog_timestamp() cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) # init_cursor should point to startup message in shard2 oplog cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman2.checkpoint, oplog_startup_ts) # No last checkpoint, no collection dump, stuff in oplog progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.collection_dump = self.opman2.collection_dump = False collection.insert({"i": 1200}) last_ts2 = self.opman2.get_last_oplog_timestamp() self.opman1.init_cursor() self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) cursor, cursor_len = self.opman2.init_cursor() for i in range(cursor_len - 1): next(cursor) self.assertEqual(next(cursor)["o"]["i"], 1200) self.assertEqual(self.opman2.checkpoint, last_ts2) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], last_ts2) # Last checkpoint exists progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress for i in range(1000): collection.insert({"i": i + 500}) entry1 = list(self.shard1_conn["local"]["oplog.rs"].find(skip=200, limit=2)) entry2 = list(self.shard2_conn["local"]["oplog.rs"].find(skip=200, limit=2)) progress.get_dict()[str(self.opman1.oplog)] = entry1[0]["ts"] progress.get_dict()[str(self.opman2.oplog)] = entry2[0]["ts"] self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.checkpoint = self.opman2.checkpoint = None cursor1, cursor_len1 = self.opman1.init_cursor() cursor2, cursor_len2 = self.opman2.init_cursor() self.assertEqual(entry1[1]["ts"], next(cursor1)["ts"]) self.assertEqual(entry2[1]["ts"], next(cursor2)["ts"]) self.assertEqual(self.opman1.checkpoint, entry1[0]["ts"]) self.assertEqual(self.opman2.checkpoint, entry2[0]["ts"]) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], entry1[0]["ts"]) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], entry2[0]["ts"]) # Last checkpoint is behind progress = LockingDict() progress.get_dict()[str(self.opman1.oplog)] = bson.Timestamp(1, 0) progress.get_dict()[str(self.opman2.oplog)] = bson.Timestamp(1, 0) self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.checkpoint = self.opman2.checkpoint = None cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman1.checkpoint) cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman2.checkpoint)
def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. no last checkpoint, no collection dump, stuff in oplog 6. last checkpoint exists 7. last checkpoint is behind """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] self.opman.collection_dump = False self.assertTrue( all(doc['op'] == 'n' for doc in self.opman.init_cursor()[0])) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman.collection_dump = True cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman.oplog = self.primary_conn['local']['oplog.rs'] collection = self.primary_conn["test"]["test"] collection.insert_one({"i": 1}) collection.delete_one({"i": 1}) time.sleep(3) last_ts = self.opman.get_last_oplog_timestamp() cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman.checkpoint, last_ts) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts) # No last checkpoint, no collection dump, something in oplog self.opman.oplog_progress = LockingDict() self.opman.collection_dump = False collection.insert_one({"i": 2}) last_ts = self.opman.get_last_oplog_timestamp() cursor, cursor_len = self.opman.init_cursor() for i in range(cursor_len - 1): next(cursor) self.assertEqual(next(cursor)['o']['i'], 2) self.assertEqual(self.opman.checkpoint, last_ts) # Last checkpoint exists progress = LockingDict() self.opman.oplog_progress = progress for i in range(1000): collection.insert_one({"i": i + 500}) entry = list(self.primary_conn["local"]["oplog.rs"].find(skip=200, limit=-2)) progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"] self.opman.oplog_progress = progress self.opman.checkpoint = None cursor, cursor_len = self.opman.init_cursor() self.assertEqual(next(cursor)["ts"], entry[1]["ts"]) self.assertEqual(self.opman.checkpoint, entry[0]["ts"]) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], entry[0]["ts"]) # Last checkpoint is behind progress = LockingDict() progress.get_dict()[str(self.opman.oplog)] = bson.Timestamp(1, 0) self.opman.oplog_progress = progress self.opman.checkpoint = None cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman.checkpoint)
class Connector(threading.Thread): """Checks the cluster for shards to tail. """ def __init__(self, address, oplog_checkpoint, target_url, ns_set, u_key, auth_key, doc_manager=None, auth_username=None, collection_dump=True, batch_size=constants.DEFAULT_BATCH_SIZE, fields=None, dest_mapping={}, auto_commit_interval=constants.DEFAULT_COMMIT_INTERVAL, continue_on_error=False): if target_url and not doc_manager: raise errors.ConnectorError("Cannot create a Connector with a " "target URL but no doc manager!") def is_string(s): try: return isinstance(s, basestring) except NameError: return isinstance(s, str) def load_doc_manager(path): name, _ = os.path.splitext(os.path.basename(path)) try: import importlib.machinery loader = importlib.machinery.SourceFileLoader(name, path) module = loader.load_module(name) except ImportError: module = imp.load_source(name, path) return module doc_manager_modules = None if doc_manager is not None: # backwards compatilibity: doc_manager may be a string if is_string(doc_manager): doc_manager_modules = [load_doc_manager(doc_manager)] # doc_manager is a list else: doc_manager_modules = [] for dm in doc_manager: doc_manager_modules.append(load_doc_manager(dm)) super(Connector, self).__init__() #can_run is set to false when we join the thread self.can_run = True #The name of the file that stores the progress of the OplogThreads self.oplog_checkpoint = oplog_checkpoint #main address - either mongos for sharded setups or a primary otherwise self.address = address #The URLs of each target system, respectively if is_string(target_url): self.target_urls = [target_url] elif target_url: self.target_urls = list(target_url) else: self.target_urls = None #The set of relevant namespaces to consider self.ns_set = ns_set #The dict of source namespace to destination namespace self.dest_mapping = dest_mapping #Whether the collection dump gracefully handles exceptions self.continue_on_error = continue_on_error #The key that is a unique document identifier for the target system. #Not necessarily the mongo unique key. self.u_key = u_key #Password for authentication self.auth_key = auth_key #Username for authentication self.auth_username = auth_username #The set of OplogThreads created self.shard_set = {} #Boolean chooses whether to dump the entire collection if no timestamp # is present in the config file self.collection_dump = collection_dump #Num entries to process before updating config file with current pos self.batch_size = batch_size #Dict of OplogThread/timestamp pairs to record progress self.oplog_progress = LockingDict() # List of fields to export self.fields = fields try: docman_kwargs = {"unique_key": u_key, "namespace_set": ns_set, "auto_commit_interval": auto_commit_interval} # No doc managers specified, using simulator if doc_manager is None: self.doc_managers = [simulator.DocManager(**docman_kwargs)] else: self.doc_managers = [] for i, d in enumerate(doc_manager_modules): # self.target_urls may be shorter than # self.doc_managers, or left as None if self.target_urls and i < len(self.target_urls): target_url = self.target_urls[i] else: target_url = None if target_url: self.doc_managers.append( d.DocManager(self.target_urls[i], **docman_kwargs)) else: self.doc_managers.append( d.DocManager(**docman_kwargs)) # If more target URLs were given than doc managers, may need # to create additional doc managers for url in self.target_urls[i + 1:]: self.doc_managers.append( doc_manager_modules[-1].DocManager(url, **docman_kwargs)) except errors.ConnectionFailed: err_msg = "MongoConnector: Could not connect to target system" logging.critical(err_msg) self.can_run = False return if self.oplog_checkpoint is not None: if not os.path.exists(self.oplog_checkpoint): info_str = ("MongoConnector: Can't find %s, " "attempting to create an empty progress log" % self.oplog_checkpoint) logging.info(info_str) try: # Create oplog progress file open(self.oplog_checkpoint, "w").close() except IOError as e: logging.critical("MongoConnector: Could not " "create a progress log: %s" % str(e)) sys.exit(2) else: if (not os.access(self.oplog_checkpoint, os.W_OK) and not os.access(self.oplog_checkpoint, os.R_OK)): logging.critical("Invalid permissions on %s! Exiting" % (self.oplog_checkpoint)) sys.exit(2) def join(self): """ Joins thread, stops it from running """ self.can_run = False for dm in self.doc_managers: dm.stop() threading.Thread.join(self) def write_oplog_progress(self): """ Writes oplog progress to file provided by user """ if self.oplog_checkpoint is None: return None # write to temp file backup_file = self.oplog_checkpoint + '.backup' os.rename(self.oplog_checkpoint, backup_file) # for each of the threads write to file with open(self.oplog_checkpoint, 'w') as dest: with self.oplog_progress as oplog_prog: oplog_dict = oplog_prog.get_dict() for oplog, time_stamp in oplog_dict.items(): oplog_str = str(oplog) timestamp = util.bson_ts_to_long(time_stamp) json_str = json.dumps([oplog_str, timestamp]) try: dest.write(json_str) except IOError: # Basically wipe the file, copy from backup dest.truncate() with open(backup_file, 'r') as backup: shutil.copyfile(backup, dest) break os.remove(self.oplog_checkpoint + '.backup') def read_oplog_progress(self): """Reads oplog progress from file provided by user. This method is only called once before any threads are spanwed. """ if self.oplog_checkpoint is None: return None # Check for empty file try: if os.stat(self.oplog_checkpoint).st_size == 0: logging.info("MongoConnector: Empty oplog progress file.") return None except OSError: return None source = open(self.oplog_checkpoint, 'r') try: data = json.load(source) except ValueError: # empty file reason = "It may be empty or corrupt." logging.info("MongoConnector: Can't read oplog progress file. %s" % (reason)) source.close() return None source.close() count = 0 oplog_dict = self.oplog_progress.get_dict() for count in range(0, len(data), 2): oplog_str = data[count] time_stamp = data[count + 1] oplog_dict[oplog_str] = util.long_to_bson_ts(time_stamp) #stored as bson_ts def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ main_conn = MongoClient(self.address) if self.auth_key is not None: main_conn['admin'].authenticate(self.auth_username, self.auth_key) self.read_oplog_progress() conn_type = None try: main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": # Make sure we are connected to a replica set is_master = main_conn.admin.command("isMaster") if not "setName" in is_master: logging.error( 'No replica set at "%s"! A replica set is required ' 'to run mongo-connector. Shutting down...' % self.address ) return # Establish a connection to the replica set as a whole main_conn.disconnect() main_conn = MongoClient(self.address, replicaSet=is_master['setName']) if self.auth_key is not None: main_conn.admin.authenticate(self.auth_username, self.auth_key) #non sharded configuration oplog_coll = main_conn['local']['oplog.rs'] oplog = OplogThread( primary_conn=main_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=False, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, repl_set=is_master['setName'], collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error ) self.shard_set[0] = oplog logging.info('MongoConnector: Starting connection thread %s' % main_conn) oplog.start() while self.can_run: if not self.shard_set[0].running: logging.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run is True: for shard_doc in main_conn['config']['shards'].find(): shard_id = shard_doc['_id'] if shard_id in self.shard_set: if not self.shard_set[shard_id].running: logging.error("MongoConnector: OplogThread " "%s unexpectedly stopped! Shutting " "down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" logging.error("MongoConnector: %s", cause) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return shard_conn = MongoClient(hosts, replicaSet=repl_set) oplog_coll = shard_conn['local']['oplog.rs'] oplog = OplogThread( primary_conn=shard_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=True, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error ) self.shard_set[shard_id] = oplog msg = "Starting connection thread" logging.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() self.oplog_thread_join() self.write_oplog_progress() def oplog_thread_join(self): """Stops all the OplogThreads """ logging.info('MongoConnector: Stopping all OplogThreads') for thread in self.shard_set.values(): thread.join()
def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. last checkpoint exists """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman1.oplog = self.shard1_conn["test"]["emptycollection"] self.opman2.oplog = self.shard2_conn["test"]["emptycollection"] self.opman1.collection_dump = False self.opman2.collection_dump = False self.assertEqual(self.opman1.init_cursor(), None) self.assertEqual(self.opman1.checkpoint, None) self.assertEqual(self.opman2.init_cursor(), None) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman1.collection_dump = True self.opman2.collection_dump = True self.assertEqual(self.opman1.init_cursor(), None) self.assertEqual(self.opman1.checkpoint, None) self.assertEqual(self.opman2.init_cursor(), None) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman1.oplog = self.shard1_conn["local"]["oplog.rs"] self.opman2.oplog = self.shard2_conn["local"]["oplog.rs"] oplog_startup_ts = self.opman2.get_last_oplog_timestamp() collection = self.mongos_conn["test"]["mcsharded"] collection.insert({"i": 1}) collection.remove({"i": 1}) time.sleep(3) last_ts1 = self.opman1.get_last_oplog_timestamp() self.assertEqual(next(self.opman1.init_cursor())["ts"], last_ts1) self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) # init_cursor should point to startup message in shard2 oplog cursor = self.opman2.init_cursor() self.assertEqual(next(cursor)["ts"], oplog_startup_ts) self.assertEqual(self.opman2.checkpoint, oplog_startup_ts) # No last checkpoint, non-empty collections, stuff in oplog progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress collection.insert({"i": 1200}) last_ts2 = self.opman2.get_last_oplog_timestamp() self.assertEqual(next(self.opman1.init_cursor())["ts"], last_ts1) self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) self.assertEqual(next(self.opman2.init_cursor())["ts"], last_ts2) self.assertEqual(self.opman2.checkpoint, last_ts2) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], last_ts2) # Last checkpoint exists progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress for i in range(1000): collection.insert({"i": i + 500}) entry1 = list( self.shard1_conn["local"]["oplog.rs"].find(skip=200, limit=2)) entry2 = list( self.shard2_conn["local"]["oplog.rs"].find(skip=200, limit=2)) progress.get_dict()[str(self.opman1.oplog)] = entry1[0]["ts"] progress.get_dict()[str(self.opman2.oplog)] = entry2[0]["ts"] self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.checkpoint = self.opman2.checkpoint = None cursor1 = self.opman1.init_cursor() cursor2 = self.opman2.init_cursor() self.assertEqual(entry1[1]["ts"], next(cursor1)["ts"]) self.assertEqual(entry2[1]["ts"], next(cursor2)["ts"]) self.assertEqual(self.opman1.checkpoint, entry1[0]["ts"]) self.assertEqual(self.opman2.checkpoint, entry2[0]["ts"]) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], entry1[0]["ts"]) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], entry2[0]["ts"])
def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. no last checkpoint, no collection dump, stuff in oplog 6. last checkpoint exists 7. last checkpoint is behind """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] self.opman.collection_dump = False self.assertTrue(all(doc["op"] == "n" for doc in self.opman.init_cursor()[0])) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman.collection_dump = True cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman.oplog = self.primary_conn["local"]["oplog.rs"] collection = self.primary_conn["test"]["test"] collection.insert({"i": 1}) collection.remove({"i": 1}) time.sleep(3) last_ts = self.opman.get_last_oplog_timestamp() cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman.checkpoint, last_ts) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts) # No last checkpoint, no collection dump, something in oplog self.opman.oplog_progress = LockingDict() self.opman.collection_dump = False collection.insert({"i": 2}) last_ts = self.opman.get_last_oplog_timestamp() cursor, cursor_len = self.opman.init_cursor() for i in range(cursor_len - 1): next(cursor) self.assertEqual(next(cursor)["o"]["i"], 2) self.assertEqual(self.opman.checkpoint, last_ts) # Last checkpoint exists progress = LockingDict() self.opman.oplog_progress = progress for i in range(1000): collection.insert({"i": i + 500}) entry = list(self.primary_conn["local"]["oplog.rs"].find(skip=200, limit=2)) progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"] self.opman.oplog_progress = progress self.opman.checkpoint = None cursor, cursor_len = self.opman.init_cursor() self.assertEqual(next(cursor)["ts"], entry[1]["ts"]) self.assertEqual(self.opman.checkpoint, entry[0]["ts"]) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], entry[0]["ts"]) # Last checkpoint is behind progress = LockingDict() progress.get_dict()[str(self.opman.oplog)] = bson.Timestamp(1, 0) self.opman.oplog_progress = progress self.opman.checkpoint = None cursor, cursor_len = self.opman.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman.checkpoint)
class Connector(threading.Thread): """Checks the cluster for shards to tail. """ def __init__(self, address, oplog_checkpoint, target_url, ns_set, u_key, auth_key, doc_manager=None, auth_username=None, collection_dump=True, batch_size=constants.DEFAULT_BATCH_SIZE, fields=None, dest_mapping={}, auto_commit_interval=constants.DEFAULT_COMMIT_INTERVAL, continue_on_error=False): if target_url and not doc_manager: raise errors.ConnectorError("Cannot create a Connector with a " "target URL but no doc manager!") def is_string(s): try: return isinstance(s, basestring) except NameError: return isinstance(s, str) def load_doc_manager(path): name, _ = os.path.splitext(os.path.basename(path)) try: import importlib.machinery loader = importlib.machinery.SourceFileLoader(name, path) module = loader.load_module(name) except ImportError: module = imp.load_source(name, path) return module doc_manager_modules = None if doc_manager is not None: # backwards compatilibity: doc_manager may be a string if is_string(doc_manager): doc_manager_modules = [load_doc_manager(doc_manager)] # doc_manager is a list else: doc_manager_modules = [] for dm in doc_manager: doc_manager_modules.append(load_doc_manager(dm)) super(Connector, self).__init__() #can_run is set to false when we join the thread self.can_run = True #The name of the file that stores the progress of the OplogThreads self.oplog_checkpoint = oplog_checkpoint #main address - either mongos for sharded setups or a primary otherwise self.address = address #The URLs of each target system, respectively if is_string(target_url): self.target_urls = [target_url] elif target_url: self.target_urls = list(target_url) else: self.target_urls = None #The set of relevant namespaces to consider self.ns_set = ns_set #The dict of source namespace to destination namespace self.dest_mapping = dest_mapping #Whether the collection dump gracefully handles exceptions self.continue_on_error = continue_on_error #The key that is a unique document identifier for the target system. #Not necessarily the mongo unique key. self.u_key = u_key #Password for authentication self.auth_key = auth_key #Username for authentication self.auth_username = auth_username #The set of OplogThreads created self.shard_set = {} #Boolean chooses whether to dump the entire collection if no timestamp # is present in the config file self.collection_dump = collection_dump #Num entries to process before updating config file with current pos self.batch_size = batch_size #Dict of OplogThread/timestamp pairs to record progress self.oplog_progress = LockingDict() # List of fields to export self.fields = fields try: docman_kwargs = { "unique_key": u_key, "namespace_set": ns_set, "auto_commit_interval": auto_commit_interval } # No doc managers specified, using simulator if doc_manager is None: self.doc_managers = [simulator.DocManager(**docman_kwargs)] else: self.doc_managers = [] for i, d in enumerate(doc_manager_modules): # self.target_urls may be shorter than # self.doc_managers, or left as None if self.target_urls and i < len(self.target_urls): target_url = self.target_urls[i] else: target_url = None if target_url: self.doc_managers.append( d.DocManager(self.target_urls[i], **docman_kwargs)) else: self.doc_managers.append(d.DocManager(**docman_kwargs)) # If more target URLs were given than doc managers, may need # to create additional doc managers for url in self.target_urls[i + 1:]: self.doc_managers.append( doc_manager_modules[-1].DocManager( url, **docman_kwargs)) except errors.ConnectionFailed: err_msg = "MongoConnector: Could not connect to target system" logging.critical(err_msg) self.can_run = False return if self.oplog_checkpoint is not None: if not os.path.exists(self.oplog_checkpoint): info_str = ("MongoConnector: Can't find %s, " "attempting to create an empty progress log" % self.oplog_checkpoint) logging.info(info_str) try: # Create oplog progress file open(self.oplog_checkpoint, "w").close() except IOError as e: logging.critical("MongoConnector: Could not " "create a progress log: %s" % str(e)) sys.exit(2) else: if (not os.access(self.oplog_checkpoint, os.W_OK) and not os.access(self.oplog_checkpoint, os.R_OK)): logging.critical("Invalid permissions on %s! Exiting" % (self.oplog_checkpoint)) sys.exit(2) def join(self): """ Joins thread, stops it from running """ self.can_run = False for dm in self.doc_managers: dm.stop() threading.Thread.join(self) def write_oplog_progress(self): """ Writes oplog progress to file provided by user """ if self.oplog_checkpoint is None: return None # write to temp file backup_file = self.oplog_checkpoint + '.backup' os.rename(self.oplog_checkpoint, backup_file) # for each of the threads write to file with open(self.oplog_checkpoint, 'w') as dest: with self.oplog_progress as oplog_prog: oplog_dict = oplog_prog.get_dict() for oplog, time_stamp in oplog_dict.items(): oplog_str = str(oplog) timestamp = util.bson_ts_to_long(time_stamp) json_str = json.dumps([oplog_str, timestamp]) try: dest.write(json_str) except IOError: # Basically wipe the file, copy from backup dest.truncate() with open(backup_file, 'r') as backup: shutil.copyfile(backup, dest) break os.remove(self.oplog_checkpoint + '.backup') def read_oplog_progress(self): """Reads oplog progress from file provided by user. This method is only called once before any threads are spanwed. """ if self.oplog_checkpoint is None: return None # Check for empty file try: if os.stat(self.oplog_checkpoint).st_size == 0: logging.info("MongoConnector: Empty oplog progress file.") return None except OSError: return None source = open(self.oplog_checkpoint, 'r') try: data = json.load(source) except ValueError: # empty file reason = "It may be empty or corrupt." logging.info("MongoConnector: Can't read oplog progress file. %s" % (reason)) source.close() return None source.close() count = 0 oplog_dict = self.oplog_progress.get_dict() for count in range(0, len(data), 2): oplog_str = data[count] time_stamp = data[count + 1] oplog_dict[oplog_str] = util.long_to_bson_ts(time_stamp) #stored as bson_ts def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ main_conn = MongoClient(self.address) if self.auth_key is not None: main_conn['admin'].authenticate(self.auth_username, self.auth_key) self.read_oplog_progress() conn_type = None try: main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": # Make sure we are connected to a replica set is_master = main_conn.admin.command("isMaster") if not "setName" in is_master: logging.error( 'No replica set at "%s"! A replica set is required ' 'to run mongo-connector. Shutting down...' % self.address) return # Establish a connection to the replica set as a whole main_conn.disconnect() main_conn = MongoClient(self.address, replicaSet=is_master['setName']) if self.auth_key is not None: main_conn.admin.authenticate(self.auth_username, self.auth_key) #non sharded configuration oplog_coll = main_conn['local']['oplog.rs'] oplog = OplogThread(primary_conn=main_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=False, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, repl_set=is_master['setName'], collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error) self.shard_set[0] = oplog logging.info('MongoConnector: Starting connection thread %s' % main_conn) oplog.start() while self.can_run: if not self.shard_set[0].running: logging.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run is True: for shard_doc in main_conn['config']['shards'].find(): shard_id = shard_doc['_id'] if shard_id in self.shard_set: if not self.shard_set[shard_id].running: logging.error("MongoConnector: OplogThread " "%s unexpectedly stopped! Shutting " "down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" logging.error("MongoConnector: %s", cause) self.oplog_thread_join() for dm in self.doc_managers: dm.stop() return shard_conn = MongoClient(hosts, replicaSet=repl_set) oplog_coll = shard_conn['local']['oplog.rs'] oplog = OplogThread( primary_conn=shard_conn, main_address=self.address, oplog_coll=oplog_coll, is_sharded=True, doc_manager=self.doc_managers, oplog_progress_dict=self.oplog_progress, namespace_set=self.ns_set, auth_key=self.auth_key, auth_username=self.auth_username, collection_dump=self.collection_dump, batch_size=self.batch_size, fields=self.fields, dest_mapping=self.dest_mapping, continue_on_error=self.continue_on_error) self.shard_set[shard_id] = oplog msg = "Starting connection thread" logging.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() self.oplog_thread_join() self.write_oplog_progress() def oplog_thread_join(self): """Stops all the OplogThreads """ logging.info('MongoConnector: Stopping all OplogThreads') for thread in self.shard_set.values(): thread.join()
def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. no last checkpoint, no collection dump, stuff in oplog 6. last checkpoint exists 7. last checkpoint is behind """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman1.oplog = self.shard1_conn["test"]["emptycollection"] self.opman2.oplog = self.shard2_conn["test"]["emptycollection"] self.opman1.collection_dump = False self.opman2.collection_dump = False self.assertTrue(all(doc['op'] == 'n' for doc in self.opman1.init_cursor()[0])) self.assertEqual(self.opman1.checkpoint, None) self.assertTrue(all(doc['op'] == 'n' for doc in self.opman2.init_cursor()[0])) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman1.collection_dump = self.opman2.collection_dump = True cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman1.checkpoint, None) cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor, None) self.assertEqual(cursor_len, 0) self.assertEqual(self.opman2.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman1.oplog = self.shard1_conn["local"]["oplog.rs"] self.opman2.oplog = self.shard2_conn["local"]["oplog.rs"] oplog_startup_ts = self.opman2.get_last_oplog_timestamp() collection = self.mongos_conn["test"]["mcsharded"] collection.insert_one({"i": 1}) collection.delete_one({"i": 1}) time.sleep(3) last_ts1 = self.opman1.get_last_oplog_timestamp() cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) # init_cursor should point to startup message in shard2 oplog cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(self.opman2.checkpoint, oplog_startup_ts) # No last checkpoint, no collection dump, stuff in oplog progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.collection_dump = self.opman2.collection_dump = False collection.insert_one({"i": 1200}) last_ts2 = self.opman2.get_last_oplog_timestamp() self.opman1.init_cursor() self.assertEqual(self.opman1.checkpoint, last_ts1) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1) cursor, cursor_len = self.opman2.init_cursor() for i in range(cursor_len - 1): next(cursor) self.assertEqual(next(cursor)["o"]["i"], 1200) self.assertEqual(self.opman2.checkpoint, last_ts2) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], last_ts2) # Last checkpoint exists progress = LockingDict() self.opman1.oplog_progress = self.opman2.oplog_progress = progress for i in range(1000): collection.insert_one({"i": i + 500}) entry1 = list( self.shard1_conn["local"]["oplog.rs"].find(skip=200, limit=-2)) entry2 = list( self.shard2_conn["local"]["oplog.rs"].find(skip=200, limit=-2)) progress.get_dict()[str(self.opman1.oplog)] = entry1[0]["ts"] progress.get_dict()[str(self.opman2.oplog)] = entry2[0]["ts"] self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.checkpoint = self.opman2.checkpoint = None cursor1, cursor_len1 = self.opman1.init_cursor() cursor2, cursor_len2 = self.opman2.init_cursor() self.assertEqual(entry1[1]["ts"], next(cursor1)["ts"]) self.assertEqual(entry2[1]["ts"], next(cursor2)["ts"]) self.assertEqual(self.opman1.checkpoint, entry1[0]["ts"]) self.assertEqual(self.opman2.checkpoint, entry2[0]["ts"]) with self.opman1.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], entry1[0]["ts"]) with self.opman2.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], entry2[0]["ts"]) # Last checkpoint is behind progress = LockingDict() progress.get_dict()[str(self.opman1.oplog)] = bson.Timestamp(1, 0) progress.get_dict()[str(self.opman2.oplog)] = bson.Timestamp(1, 0) self.opman1.oplog_progress = self.opman2.oplog_progress = progress self.opman1.checkpoint = self.opman2.checkpoint = None cursor, cursor_len = self.opman1.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman1.checkpoint) cursor, cursor_len = self.opman2.init_cursor() self.assertEqual(cursor_len, 0) self.assertEqual(cursor, None) self.assertIsNotNone(self.opman2.checkpoint)
def test_init_cursor(self): """Test the init_cursor method Cases: 1. no last checkpoint, no collection dump 2. no last checkpoint, collection dump ok and stuff to dump 3. no last checkpoint, nothing to dump, stuff in oplog 4. no last checkpoint, nothing to dump, nothing in oplog 5. last checkpoint exists """ # N.B. these sub-cases build off of each other and cannot be re-ordered # without side-effects # No last checkpoint, no collection dump, nothing in oplog # "change oplog collection" to put nothing in oplog self.opman.oplog = self.primary_conn["test"]["emptycollection"] self.opman.collection_dump = False self.assertEqual(self.opman.init_cursor(), None) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, nothing in oplog self.opman.collection_dump = True self.assertEqual(self.opman.init_cursor(), None) self.assertEqual(self.opman.checkpoint, None) # No last checkpoint, empty collections, something in oplog self.opman.oplog = self.primary_conn['local']['oplog.rs'] collection = self.primary_conn["test"]["test"] collection.insert({"i": 1}) collection.remove({"i": 1}) time.sleep(3) last_ts = self.opman.get_last_oplog_timestamp() self.assertEqual(next(self.opman.init_cursor())["ts"], last_ts) self.assertEqual(self.opman.checkpoint, last_ts) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts) # No last checkpoint, non-empty collections, stuff in oplog self.opman.oplog_progress = LockingDict() self.assertEqual(next(self.opman.init_cursor())["ts"], last_ts) self.assertEqual(self.opman.checkpoint, last_ts) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts) # Last checkpoint exists progress = LockingDict() self.opman.oplog_progress = progress for i in range(1000): collection.insert({"i": i + 500}) entry = list( self.primary_conn["local"]["oplog.rs"].find(skip=200, limit=2)) progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"] self.opman.oplog_progress = progress self.opman.checkpoint = None cursor = self.opman.init_cursor() self.assertEqual(entry[1]["ts"], next(cursor)["ts"]) self.assertEqual(self.opman.checkpoint, entry[0]["ts"]) with self.opman.oplog_progress as prog: self.assertEqual(prog.get_dict()[str(self.opman.oplog)], entry[0]["ts"])
class Connector(threading.Thread): """Checks the cluster for shards to tail. """ def __init__(self, address, oplog_checkpoint, target_url, ns_set, u_key, auth_key, doc_manager=None, auth_username=None): if doc_manager is not None: doc_manager = imp.load_source('DocManager', doc_manager) else: from mongo_connector.doc_manager import DocManager time.sleep(1) super(Connector, self).__init__() #can_run is set to false when we join the thread self.can_run = True #The name of the file that stores the progress of the OplogThreads self.oplog_checkpoint = oplog_checkpoint #main address - either mongos for sharded setups or a primary otherwise self.address = address #The URL of the target system self.target_url = target_url #The set of relevant namespaces to consider self.ns_set = ns_set #The key that is a unique document identifier for the target system. #Not necessarily the mongo unique key. self.u_key = u_key #Password for authentication self.auth_key = auth_key #Username for authentication self.auth_username = auth_username #The set of OplogThreads created self.shard_set = {} #Dict of OplogThread/timestamp pairs to record progress self.oplog_progress = LockingDict() try: if target_url is None: if doc_manager is None: # imported using from... import self.doc_manager = DocManager(unique_key=u_key) else: # imported using load source self.doc_manager = doc_manager.DocManager(unique_key=u_key) else: if doc_manager is None: self.doc_manager = DocManager(self.target_url, unique_key=u_key) else: self.doc_manager = doc_manager.DocManager(self.target_url, unique_key=u_key) except errors.ConnectionFailed: err_msg = "MongoConnector: Could not connect to target system" logging.critical(err_msg) self.can_run = False return if self.oplog_checkpoint is not None: if not os.path.exists(self.oplog_checkpoint): info_str = ("MongoConnector: Can't find %s, " "attempting to create an empty progress log" % self.oplog_checkpoint) logging.info(info_str) try: # Create oplog progress file open(self.oplog_checkpoint, "w").close() except IOError as e: logging.critical("MongoConnector: Could not " "create a progress log: %s" % str(e)) sys.exit(1) else: if (not os.access(self.oplog_checkpoint, os.W_OK) and not os.access(self.oplog_checkpoint, os.R_OK )): logging.critical("Invalid permissions on %s! Exiting" % (self.oplog_checkpoint)) sys.exit(1) def join(self): """ Joins thread, stops it from running """ self.can_run = False self.doc_manager.stop() threading.Thread.join(self) def write_oplog_progress(self): """ Writes oplog progress to file provided by user """ if self.oplog_checkpoint is None: return None # write to temp file backup_file = self.oplog_checkpoint + '.backup' os.rename(self.oplog_checkpoint, backup_file) # for each of the threads write to file with open(self.oplog_checkpoint, 'w') as dest: with self.oplog_progress as oplog_prog: oplog_dict = oplog_prog.get_dict() for oplog, time_stamp in oplog_dict.items(): oplog_str = str(oplog) timestamp = util.bson_ts_to_long(time_stamp) json_str = json.dumps([oplog_str, timestamp]) try: dest.write(json_str) except IOError: # Basically wipe the file, copy from backup dest.truncate() with open(backup_file, 'r') as backup: shutil.copyfile(backup, dest) break os.remove(self.oplog_checkpoint + '.backup') def read_oplog_progress(self): """Reads oplog progress from file provided by user. This method is only called once before any threads are spanwed. """ if self.oplog_checkpoint is None: return None # Check for empty file try: if os.stat(self.oplog_checkpoint).st_size == 0: logging.info("MongoConnector: Empty oplog progress file.") return None except OSError: return None source = open(self.oplog_checkpoint, 'r') try: data = json.load(source) except ValueError: # empty file reason = "It may be empty or corrupt." logging.info("MongoConnector: Can't read oplog progress file. %s" % (reason)) source.close() return None source.close() count = 0 oplog_dict = self.oplog_progress.get_dict() for count in range(0, len(data), 2): oplog_str = data[count] time_stamp = data[count + 1] oplog_dict[oplog_str] = util.long_to_bson_ts(time_stamp) #stored as bson_ts def run(self): """Discovers the mongo cluster and creates a thread for each primary. """ main_conn = Connection(self.address) if self.auth_key is not None: main_conn['admin'].authenticate(self.auth_username, self.auth_key) self.read_oplog_progress() conn_type = None try: main_conn.admin.command("isdbgrid") except pymongo.errors.OperationFailure: conn_type = "REPLSET" if conn_type == "REPLSET": #non sharded configuration oplog_coll = main_conn['local']['oplog.rs'] prim_admin = main_conn.admin repl_set = prim_admin.command("replSetGetStatus")['set'] oplog = oplog_manager.OplogThread(main_conn, (main_conn.host + ":" + str(main_conn.port)), oplog_coll, False, self.doc_manager, self.oplog_progress, self.ns_set, self.auth_key, self.auth_username, repl_set=repl_set) self.shard_set[0] = oplog logging.info('MongoConnector: Starting connection thread %s' % main_conn) oplog.start() while self.can_run: if not self.shard_set[0].running: logging.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[0]))) self.oplog_thread_join() self.doc_manager.stop() return self.write_oplog_progress() time.sleep(1) else: # sharded cluster while self.can_run is True: for shard_doc in main_conn['config']['shards'].find(): shard_id = shard_doc['_id'] if shard_id in self.shard_set: if not self.shard_set[shard_id].running: logging.error("MongoConnector: OplogThread" " %s unexpectedly stopped! Shutting down" % (str(self.shard_set[shard_id]))) self.oplog_thread_join() self.doc_manager.stop() return self.write_oplog_progress() time.sleep(1) continue try: repl_set, hosts = shard_doc['host'].split('/') except ValueError: cause = "The system only uses replica sets!" logging.error("MongoConnector: %s", cause) self.oplog_thread_join() self.doc_manager.stop() return shard_conn = Connection(hosts, replicaset=repl_set) oplog_coll = shard_conn['local']['oplog.rs'] oplog = oplog_manager.OplogThread(shard_conn, self.address, oplog_coll, True, self.doc_manager, self.oplog_progress, self.ns_set, self.auth_key, self.auth_username) self.shard_set[shard_id] = oplog msg = "Starting connection thread" logging.info("MongoConnector: %s %s" % (msg, shard_conn)) oplog.start() self.oplog_thread_join() self.write_oplog_progress() def oplog_thread_join(self): """Stops all the OplogThreads """ logging.info('MongoConnector: Stopping all OplogThreads') for thread in self.shard_set.values(): thread.join()