def write_oplog_progress(self): """ Writes oplog progress to file provided by user """ if self.oplog_checkpoint is None: return None # write to temp file backup_file = self.oplog_checkpoint + '.backup' os.rename(self.oplog_checkpoint, backup_file) # for each of the threads write to file with open(self.oplog_checkpoint, 'w') as dest: with self.oplog_progress as oplog_prog: oplog_dict = oplog_prog.get_dict() for oplog, ts in oplog_dict.items(): oplog_str = str(oplog) timestamp = util.bson_ts_to_long(ts) json_str = json.dumps([oplog_str, timestamp]) try: dest.write(json_str) except IOError: # Basically wipe the file, copy from backup dest.truncate() with open(backup_file, 'r') as backup: shutil.copyfile(backup, dest) break os.remove(self.oplog_checkpoint + '.backup')
def run(self): """Start the oplog worker. """ while self.running is True: cursor = self.init_cursor() # we've fallen too far behind if cursor is None and self.checkpoint is not None: err_msg = "OplogManager: Last entry no longer in oplog" effect = "cannot recover!" logging.error('%s %s %s' % (err_msg, effect, self.oplog)) self.running = False continue #The only entry is the last one we processed if util.retry_until_ok(cursor.count) == 1: time.sleep(1) continue last_ts = None err = False try: for entry in cursor: if 'fromMigrate' in entry and entry['fromMigrate'] is True: continue #sync the current oplog operation operation = entry['op'] ns = entry['ns'] #check if ns is excluded or not. #also ensure non-empty namespace set. if ns not in self.namespace_set and self.namespace_set: continue #delete if operation == 'd': entry['_id'] = entry['o']['_id'] self.doc_manager.remove(entry) #insert/update. They are equal because of lack of support #for partial update elif operation == 'i' or operation == 'u': doc = self.retrieve_doc(entry) if doc is not None: doc['_ts'] = util.bson_ts_to_long(entry['ts']) doc['ns'] = ns self.doc_manager.upsert(doc) last_ts = entry['ts'] except (pymongo.errors.AutoReconnect, pymongo.errors.OperationFailure): err = True pass if err is True and self.auth_key is not None: primary_conn['admin'].authenticate(self.auth_username, self.auth_key) err = False if last_ts is not None: self.checkpoint = last_ts self.update_checkpoint()
def print_oplog_last_ts(host, origin, ns, destination, user=None, password=None): """Simply just print into a file the last timestamp """ #newst oplog entry newst_oplog = 0 #get all shards from target url target_shards = get_shard_nodes(host) #go to all nodes of the shard and get the newst oplog operation. for shard_doc in target_shards: repl_set, hosts = shard_doc['host'].split('/') shard_connection = MongoClient(hosts, replicaSet=repl_set) local = shard_connection['local'] oplog = local['oplog.rs'] #collect the last timestamp for the oplog which is not a migration operation last_oplog_entry = oplog.find_one( { 'ns': ns, 'fromMigrate': { '$exists': 0 } }, sort=[('$natural', pymongo.DESCENDING)]) #check if the is a last entry timestamp = util.bson_ts_to_long(last_oplog_entry['ts']) if timestamp > newst_oplog: newst_oplog = timestamp #get all shards from target url origin_shards = get_shard_nodes(origin) with open(destination, 'w') as dest_file: jsons = [] for shard_doc in origin_shards: repl_set, hosts = shard_doc['host'].split('/') shard_connection = MongoClient(hosts, replicaSet=repl_set) local = shard_connection['local'] oplog = local['oplog.rs'] oplog_str = str(oplog) jsons.append([oplog_str, newst_oplog]) try: json_str = json.dumps(jsons) dest_file.write(json_str) print json_str except IOError: dest_file.truncate() print "Problem writing to file %s" % destination dest_file.close() validate_destination_file(destination)
def test_bson_ts_to_long(self): """Test bson_ts_to_long and long_to_bson_ts """ ts = timestamp.Timestamp(0x12345678, 0x90abcdef) self.assertEqual(0x1234567890abcdef, bson_ts_to_long(ts)) self.assertEqual(long_to_bson_ts(0x1234567890abcdef), ts) print("PASSED BSON TS TO LONG")
def test_bson_ts_to_long(self): """Test bson_ts_to_long and long_to_bson_ts """ tstamp = timestamp.Timestamp(0x12345678, 0x90abcdef) self.assertEqual(0x1234567890abcdef, bson_ts_to_long(tstamp)) self.assertEqual(long_to_bson_ts(0x1234567890abcdef), tstamp)
def dump_collection(self): """Dumps collection into the target system. Returns the timestamp of the last oplog entry right before it starts dumping. This method is called when we're initializing the cursor and have no configs i.e. when we're starting for the first time. """ dump_set = self.namespace_set #no namespaces specified if not self.namespace_set: db_list = self.main_connection.database_names() for db in db_list: if db == "config" or db == "local": continue coll_list = self.main_connection[db].collection_names() for coll in coll_list: if coll.startswith("system"): continue namespace = str(db) + "." + str(coll) dump_set.append(namespace) timestamp = util.retry_until_ok(self.get_last_oplog_timestamp) if timestamp is None: return None long_ts = util.bson_ts_to_long(timestamp) for namespace in dump_set: db, coll = namespace.split('.', 1) target_coll = self.main_connection[db][coll] cursor = util.retry_until_ok(target_coll.find) try: for doc in cursor: doc['ns'] = namespace doc['_ts'] = long_ts self.doc_manager.upsert(doc) except pymongo.errors.AutoReconnect as e: err_msg = "OplogManager: Failed during dump collection. " err_msg += "AutoReconnect error: %s." % e effect = " Cannot recover!" logging.error('%s %s %s' % (err_msg, effect, self.oplog)) self.running = False return except pymongo.errors.OperationFailure as e: err_msg = "OplogManager: Failed during dump collection" err_msg += "OperationFailure error: %s." % e effect = " Cannot recover!" logging.error('%s %s %s' % (err_msg, effect, self.oplog)) self.running = False return return timestamp
def dump_collection(self): """Dumps collection into the target system. This method is called when we're initializing the cursor and have no configs i.e. when we're starting for the first time. """ dump_set = self.namespace_set #no namespaces specified if not self.namespace_set: db_list = self.main_connection.database_names() for database in db_list: if database == "config" or database == "local": continue coll_list = self.main_connection[database].collection_names() for coll in coll_list: if coll.startswith("system"): continue namespace = str(database) + "." + str(coll) dump_set.append(namespace) timestamp = util.retry_until_ok(self.get_last_oplog_timestamp) if timestamp is None: return None for namespace in dump_set: database, coll = namespace.split('.', 1) target_coll = self.main_connection[database][coll] cursor = util.retry_until_ok(target_coll.find) long_ts = util.bson_ts_to_long(timestamp) try: for doc in cursor: doc['ns'] = namespace doc['_ts'] = long_ts try: self.doc_manager.upsert(doc) except SystemError: logging.error("Unable to insert %s" % (doc)) except (pymongo.errors.AutoReconnect, pymongo.errors.OperationFailure): err_msg = "OplogManager: Failed during dump collection" effect = "cannot recover!" logging.error('%s %s %s' % (err_msg, effect, self.oplog)) self.running = False return return timestamp
def print_oplog_last_ts(host, origin, ns, destination, user=None, password=None): """Simply just print into a file the last timestamp """ # newst oplog entry newst_oplog = 0 # get all shards from target url target_shards = get_shard_nodes(host) # go to all nodes of the shard and get the newst oplog operation. for shard_doc in target_shards: repl_set, hosts = shard_doc["host"].split("/") shard_connection = MongoClient(hosts, replicaSet=repl_set) local = shard_connection["local"] oplog = local["oplog.rs"] # collect the last timestamp for the oplog which is not a migration operation last_oplog_entry = oplog.find_one( {"ns": ns, "fromMigrate": {"$exists": 0}}, sort=[("$natural", pymongo.DESCENDING)] ) # check if the is a last entry timestamp = util.bson_ts_to_long(last_oplog_entry["ts"]) if timestamp > newst_oplog: newst_oplog = timestamp # get all shards from target url origin_shards = get_shard_nodes(origin) with open(destination, "w") as dest_file: jsons = [] for shard_doc in origin_shards: repl_set, hosts = shard_doc["host"].split("/") shard_connection = MongoClient(hosts, replicaSet=repl_set) local = shard_connection["local"] oplog = local["oplog.rs"] oplog_str = str(oplog) jsons.append([oplog_str, newst_oplog]) try: json_str = json.dumps(jsons) dest_file.write(json_str) print json_str except IOError: dest_file.truncate() print "Problem writing to file %s" % destination dest_file.close() validate_destination_file(destination)
def rollback(self): """Rollback target system to consistent state. The strategy is to find the latest timestamp in the target system and the largest timestamp in the oplog less than the latest target system timestamp. This defines the rollback window and we just roll these back until the oplog and target system are in consistent states. """ self.doc_manager.commit() last_inserted_doc = self.doc_manager.get_last_doc() if last_inserted_doc is None: return None target_ts = util.long_to_bson_ts(last_inserted_doc['_ts']) last_oplog_entry = self.oplog.find_one({'ts': {'$lte': target_ts}}, sort=[('$natural', pymongo.DESCENDING)]) if last_oplog_entry is None: return None rollback_cutoff_ts = last_oplog_entry['ts'] start_ts = util.bson_ts_to_long(rollback_cutoff_ts) end_ts = last_inserted_doc['_ts'] rollback_set = {} # this is a dictionary of ns:list of docs for doc in self.doc_manager.search(start_ts, end_ts): if doc['ns'] in rollback_set: rollback_set[doc['ns']].append(doc) else: rollback_set[doc['ns']] = [doc] for namespace, doc_list in rollback_set.items(): database, coll = namespace.split('.', 1) obj_id = bson.objectid.ObjectId bson_obj_id_list = [obj_id(doc['_id']) for doc in doc_list] to_update = util.retry_until_ok( self.main_connection[database][coll].find, {'_id': {'$in': bson_obj_id_list}}) #doc list are docs in target system, to_update are docs in mongo doc_hash = {} # hash by _id for doc in doc_list: doc_hash[bson.objectid.ObjectId(doc['_id'])] = doc to_index = [] count = 0 while True: try: for doc in to_update: if doc['_id'] in doc_hash: del doc_hash[doc['_id']] to_index.append(doc) break except (pymongo.errors.OperationFailure, pymongo.errors.AutoReconnect): count += 1 if count > 60: sys.exit(1) time.sleep(1) #delete the inconsistent documents for doc in doc_hash.values(): self.doc_manager.remove(doc) #insert the ones from mongo for doc in to_index: doc['_ts'] = util.bson_ts_to_long(rollback_cutoff_ts) doc['ns'] = namespace try: self.doc_manager.upsert(doc) except SystemError: logging.error("Unable to insert %s" % (doc)) return rollback_cutoff_ts
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') start_cluster() test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') obj1 = ObjectId('4ff74db3f646462b38000001') mongos['test']['test'].remove({}) mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'}, safe=1) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': obj1} #try kill one, try restarting killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: current_conn = mongos['test']['test'] current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1) break except: count += 1 if count > 60: string = 'Call to insert doc failed too many times' logging.error(string) sys.exit(1) time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): print(mongos['test']['test'].find().count()) time.sleep(1) killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY']) startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': obj2} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert(len(results) == 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts)) #test_oplog.join() print("PASSED TEST ROLLBACK")
def dump_collection(self): """Dumps collection into the target system. This method is called when we're initializing the cursor and have no configs i.e. when we're starting for the first time. """ dump_set = self.namespace_set #no namespaces specified if not self.namespace_set: db_list = self.main_connection.database_names() for db in db_list: if db == "config" or db == "local": continue coll_list = self.main_connection[db].collection_names() for coll in coll_list: if coll.startswith("system"): continue namespace = str(db) + "." + str(coll) dump_set.append(namespace) long_ts = None for namespace in dump_set: db, coll = namespace.split('.', 1) target_coll = self.main_connection[db][coll] cursor = util.retry_until_ok(target_coll.find) cursor = cursor.sort('$natural', pymongo.DESCENDING) oplog_cursor = util.retry_until_ok(self.oplog.find) oplog_cursor = oplog_cursor.sort('$natural', pymongo.DESCENDING) for entry in oplog_cursor: if entry['op'] != 'i': continue #The 'o' field represents the document search_doc = entry['o'] cursor.rewind() for doc in cursor: if search_doc == doc: long_ts = util.bson_ts_to_long(entry['ts']) break if long_ts: break cursor.rewind() try: for doc in cursor: doc['ns'] = namespace doc['_ts'] = long_ts self.doc_manager.upsert(doc) except (pymongo.errors.AutoReconnect, pymongo.errors.OperationFailure): err_msg = "OplogManager: Failed during dump collection" effect = "cannot recover!" logging.error('%s %s %s' % (err_msg, effect, self.oplog)) self.running = False return if long_ts: long_ts = util.long_to_bson_ts(long_ts) else: # Implies that we are just initiating the set long_ts = self.get_last_oplog_timestamp() return long_ts
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') start_cluster() test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') obj1 = ObjectId('4ff74db3f646462b38000001') mongos['test']['test'].remove({}) mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'}, safe=1) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = { 'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': obj1 } #try kill one, try restarting killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: current_conn = mongos['test']['test'] current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1) break except: count += 1 if count > 60: string = 'Call to insert doc failed too many times' logging.error(string) sys.exit(1) time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): print(mongos['test']['test'].find().count()) time.sleep(1) killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY']) startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = { 'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': obj2 } test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert (len(results) == 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts)) #test_oplog.join() print("PASSED TEST ROLLBACK")
def rollback(self): """Rollback target system to consistent state. The strategy is to find the latest timestamp in the target system and the largest timestamp in the oplog less than the latest target system timestamp. This defines the rollback window and we just roll these back until the oplog and target system are in consistent states. """ self.doc_manager.commit() last_inserted_doc = self.doc_manager.get_last_doc() if last_inserted_doc is None: return None target_ts = util.long_to_bson_ts(last_inserted_doc['_ts']) last_oplog_entry = self.oplog.find_one({'ts': { '$lte': target_ts }}, sort=[('$natural', pymongo.DESCENDING)]) if last_oplog_entry is None: return None rollback_cutoff_ts = last_oplog_entry['ts'] start_ts = util.bson_ts_to_long(rollback_cutoff_ts) end_ts = last_inserted_doc['_ts'] rollback_set = {} # this is a dictionary of ns:list of docs for doc in self.doc_manager.search(start_ts, end_ts): if doc['ns'] in rollback_set: rollback_set[doc['ns']].append(doc) else: rollback_set[doc['ns']] = [doc] for namespace, doc_list in rollback_set.items(): database, coll = namespace.split('.', 1) obj_id = bson.objectid.ObjectId bson_obj_id_list = [obj_id(doc['_id']) for doc in doc_list] to_update = util.retry_until_ok( self.main_connection[database][coll].find, {'_id': { '$in': bson_obj_id_list }}) #doc list are docs in target system, to_update are docs in mongo doc_hash = {} # hash by _id for doc in doc_list: doc_hash[bson.objectid.ObjectId(doc['_id'])] = doc to_index = [] count = 0 while True: try: for doc in to_update: if doc['_id'] in doc_hash: del doc_hash[doc['_id']] to_index.append(doc) break except (pymongo.errors.OperationFailure, pymongo.errors.AutoReconnect): count += 1 if count > 60: sys.exit(1) time.sleep(1) #delete the inconsistent documents for doc in doc_hash.values(): self.doc_manager.remove(doc) #insert the ones from mongo for doc in to_index: doc['_ts'] = util.bson_ts_to_long(rollback_cutoff_ts) doc['ns'] = namespace try: self.doc_manager.upsert(doc) except SystemError: logging.error("Unable to insert %s" % (doc)) return rollback_cutoff_ts