def write_oplog_progress(self):
""" Writes oplog progress to file provided by user
"""
if self.oplog_checkpoint is None:
return None
# write to temp file
backup_file = self.oplog_checkpoint + '.backup'
os.rename(self.oplog_checkpoint, backup_file)
# for each of the threads write to file
with open(self.oplog_checkpoint, 'w') as dest:
with self.oplog_progress as oplog_prog:
oplog_dict = oplog_prog.get_dict()
for oplog, ts in oplog_dict.items():
oplog_str = str(oplog)
timestamp = util.bson_ts_to_long(ts)
json_str = json.dumps([oplog_str, timestamp])
try:
dest.write(json_str)
except IOError:
# Basically wipe the file, copy from backup
dest.truncate()
with open(backup_file, 'r') as backup:
shutil.copyfile(backup, dest)
break
os.remove(self.oplog_checkpoint + '.backup')
def run(self):
"""Start the oplog worker.
"""
while self.running is True:
cursor = self.init_cursor()
# we've fallen too far behind
if cursor is None and self.checkpoint is not None:
err_msg = "OplogManager: Last entry no longer in oplog"
effect = "cannot recover!"
logging.error('%s %s %s' % (err_msg, effect, self.oplog))
self.running = False
continue
#The only entry is the last one we processed
if util.retry_until_ok(cursor.count) == 1:
time.sleep(1)
continue
last_ts = None
err = False
try:
for entry in cursor:
if 'fromMigrate' in entry and entry['fromMigrate'] is True:
continue
#sync the current oplog operation
operation = entry['op']
ns = entry['ns']
#check if ns is excluded or not.
#also ensure non-empty namespace set.
if ns not in self.namespace_set and self.namespace_set:
continue
#delete
if operation == 'd':
entry['_id'] = entry['o']['_id']
self.doc_manager.remove(entry)
#insert/update. They are equal because of lack of support
#for partial update
elif operation == 'i' or operation == 'u':
doc = self.retrieve_doc(entry)
if doc is not None:
doc['_ts'] = util.bson_ts_to_long(entry['ts'])
doc['ns'] = ns
self.doc_manager.upsert(doc)
last_ts = entry['ts']
except (pymongo.errors.AutoReconnect,
pymongo.errors.OperationFailure):
err = True
pass
if err is True and self.auth_key is not None:
primary_conn['admin'].authenticate(self.auth_username,
self.auth_key)
err = False
if last_ts is not None:
self.checkpoint = last_ts
self.update_checkpoint()
def write_oplog_progress(self):
""" Writes oplog progress to file provided by user
"""
if self.oplog_checkpoint is None:
return None
# write to temp file
backup_file = self.oplog_checkpoint + '.backup'
os.rename(self.oplog_checkpoint, backup_file)
# for each of the threads write to file
with open(self.oplog_checkpoint, 'w') as dest:
with self.oplog_progress as oplog_prog:
oplog_dict = oplog_prog.get_dict()
for oplog, ts in oplog_dict.items():
oplog_str = str(oplog)
timestamp = util.bson_ts_to_long(ts)
json_str = json.dumps([oplog_str, timestamp])
try:
dest.write(json_str)
except IOError:
# Basically wipe the file, copy from backup
dest.truncate()
with open(backup_file, 'r') as backup:
shutil.copyfile(backup, dest)
break
os.remove(self.oplog_checkpoint + '.backup')
def print_oplog_last_ts(host,
origin,
ns,
destination,
user=None,
password=None):
"""Simply just print into a file the last timestamp
"""
#newst oplog entry
newst_oplog = 0
#get all shards from target url
target_shards = get_shard_nodes(host)
#go to all nodes of the shard and get the newst oplog operation.
for shard_doc in target_shards:
repl_set, hosts = shard_doc['host'].split('/')
shard_connection = MongoClient(hosts, replicaSet=repl_set)
local = shard_connection['local']
oplog = local['oplog.rs']
#collect the last timestamp for the oplog which is not a migration operation
last_oplog_entry = oplog.find_one(
{
'ns': ns,
'fromMigrate': {
'$exists': 0
}
},
sort=[('$natural', pymongo.DESCENDING)])
#check if the is a last entry
timestamp = util.bson_ts_to_long(last_oplog_entry['ts'])
if timestamp > newst_oplog:
newst_oplog = timestamp
#get all shards from target url
origin_shards = get_shard_nodes(origin)
with open(destination, 'w') as dest_file:
jsons = []
for shard_doc in origin_shards:
repl_set, hosts = shard_doc['host'].split('/')
shard_connection = MongoClient(hosts, replicaSet=repl_set)
local = shard_connection['local']
oplog = local['oplog.rs']
oplog_str = str(oplog)
jsons.append([oplog_str, newst_oplog])
try:
json_str = json.dumps(jsons)
dest_file.write(json_str)
print json_str
except IOError:
dest_file.truncate()
print "Problem writing to file %s" % destination
dest_file.close()
validate_destination_file(destination)
def test_bson_ts_to_long(self):
"""Test bson_ts_to_long and long_to_bson_ts
"""
ts = timestamp.Timestamp(0x12345678, 0x90abcdef)
self.assertEqual(0x1234567890abcdef, bson_ts_to_long(ts))
self.assertEqual(long_to_bson_ts(0x1234567890abcdef), ts)
print("PASSED BSON TS TO LONG")
def test_bson_ts_to_long(self):
"""Test bson_ts_to_long and long_to_bson_ts
"""
ts = timestamp.Timestamp(0x12345678, 0x90abcdef)
self.assertEqual(0x1234567890abcdef, bson_ts_to_long(ts))
self.assertEqual(long_to_bson_ts(0x1234567890abcdef), ts)
print("PASSED BSON TS TO LONG")
def test_bson_ts_to_long(self):
"""Test bson_ts_to_long and long_to_bson_ts
"""
tstamp = timestamp.Timestamp(0x12345678, 0x90abcdef)
self.assertEqual(0x1234567890abcdef,
bson_ts_to_long(tstamp))
self.assertEqual(long_to_bson_ts(0x1234567890abcdef),
tstamp)
def dump_collection(self):
"""Dumps collection into the target system. Returns the timestamp
of the last oplog entry right before it starts dumping.
This method is called when we're initializing the cursor and have no
configs i.e. when we're starting for the first time.
"""
dump_set = self.namespace_set
#no namespaces specified
if not self.namespace_set:
db_list = self.main_connection.database_names()
for db in db_list:
if db == "config" or db == "local":
continue
coll_list = self.main_connection[db].collection_names()
for coll in coll_list:
if coll.startswith("system"):
continue
namespace = str(db) + "." + str(coll)
dump_set.append(namespace)
timestamp = util.retry_until_ok(self.get_last_oplog_timestamp)
if timestamp is None:
return None
long_ts = util.bson_ts_to_long(timestamp)
for namespace in dump_set:
db, coll = namespace.split('.', 1)
target_coll = self.main_connection[db][coll]
cursor = util.retry_until_ok(target_coll.find)
try:
for doc in cursor:
doc['ns'] = namespace
doc['_ts'] = long_ts
self.doc_manager.upsert(doc)
except pymongo.errors.AutoReconnect as e:
err_msg = "OplogManager: Failed during dump collection. "
err_msg += "AutoReconnect error: %s." % e
effect = " Cannot recover!"
logging.error('%s %s %s' % (err_msg, effect, self.oplog))
self.running = False
return
except pymongo.errors.OperationFailure as e:
err_msg = "OplogManager: Failed during dump collection"
err_msg += "OperationFailure error: %s." % e
effect = " Cannot recover!"
logging.error('%s %s %s' % (err_msg, effect, self.oplog))
self.running = False
return
return timestamp
def dump_collection(self):
"""Dumps collection into the target system.
This method is called when we're initializing the cursor and have no
configs i.e. when we're starting for the first time.
"""
dump_set = self.namespace_set
#no namespaces specified
if not self.namespace_set:
db_list = self.main_connection.database_names()
for database in db_list:
if database == "config" or database == "local":
continue
coll_list = self.main_connection[database].collection_names()
for coll in coll_list:
if coll.startswith("system"):
continue
namespace = str(database) + "." + str(coll)
dump_set.append(namespace)
timestamp = util.retry_until_ok(self.get_last_oplog_timestamp)
if timestamp is None:
return None
for namespace in dump_set:
database, coll = namespace.split('.', 1)
target_coll = self.main_connection[database][coll]
cursor = util.retry_until_ok(target_coll.find)
long_ts = util.bson_ts_to_long(timestamp)
try:
for doc in cursor:
doc['ns'] = namespace
doc['_ts'] = long_ts
try:
self.doc_manager.upsert(doc)
except SystemError:
logging.error("Unable to insert %s" % (doc))
except (pymongo.errors.AutoReconnect,
pymongo.errors.OperationFailure):
err_msg = "OplogManager: Failed during dump collection"
effect = "cannot recover!"
logging.error('%s %s %s' % (err_msg, effect, self.oplog))
self.running = False
return
return timestamp
def print_oplog_last_ts(host, origin, ns, destination, user=None, password=None):
"""Simply just print into a file the last timestamp
"""
# newst oplog entry
newst_oplog = 0
# get all shards from target url
target_shards = get_shard_nodes(host)
# go to all nodes of the shard and get the newst oplog operation.
for shard_doc in target_shards:
repl_set, hosts = shard_doc["host"].split("/")
shard_connection = MongoClient(hosts, replicaSet=repl_set)
local = shard_connection["local"]
oplog = local["oplog.rs"]
# collect the last timestamp for the oplog which is not a migration operation
last_oplog_entry = oplog.find_one(
{"ns": ns, "fromMigrate": {"$exists": 0}}, sort=[("$natural", pymongo.DESCENDING)]
)
# check if the is a last entry
timestamp = util.bson_ts_to_long(last_oplog_entry["ts"])
if timestamp > newst_oplog:
newst_oplog = timestamp
# get all shards from target url
origin_shards = get_shard_nodes(origin)
with open(destination, "w") as dest_file:
jsons = []
for shard_doc in origin_shards:
repl_set, hosts = shard_doc["host"].split("/")
shard_connection = MongoClient(hosts, replicaSet=repl_set)
local = shard_connection["local"]
oplog = local["oplog.rs"]
oplog_str = str(oplog)
jsons.append([oplog_str, newst_oplog])
try:
json_str = json.dumps(jsons)
dest_file.write(json_str)
print json_str
except IOError:
dest_file.truncate()
print "Problem writing to file %s" % destination
dest_file.close()
validate_destination_file(destination)
def rollback(self):
"""Rollback target system to consistent state.
The strategy is to find the latest timestamp in the target system and
the largest timestamp in the oplog less than the latest target system
timestamp. This defines the rollback window and we just roll these
back until the oplog and target system are in consistent states.
"""
self.doc_manager.commit()
last_inserted_doc = self.doc_manager.get_last_doc()
if last_inserted_doc is None:
return None
target_ts = util.long_to_bson_ts(last_inserted_doc['_ts'])
last_oplog_entry = self.oplog.find_one({'ts': {'$lte': target_ts}},
sort=[('$natural',
pymongo.DESCENDING)])
if last_oplog_entry is None:
return None
rollback_cutoff_ts = last_oplog_entry['ts']
start_ts = util.bson_ts_to_long(rollback_cutoff_ts)
end_ts = last_inserted_doc['_ts']
rollback_set = {} # this is a dictionary of ns:list of docs
for doc in self.doc_manager.search(start_ts, end_ts):
if doc['ns'] in rollback_set:
rollback_set[doc['ns']].append(doc)
else:
rollback_set[doc['ns']] = [doc]
for namespace, doc_list in rollback_set.items():
database, coll = namespace.split('.', 1)
obj_id = bson.objectid.ObjectId
bson_obj_id_list = [obj_id(doc['_id']) for doc in doc_list]
to_update = util.retry_until_ok(
self.main_connection[database][coll].find,
{'_id': {'$in': bson_obj_id_list}})
#doc list are docs in target system, to_update are docs in mongo
doc_hash = {} # hash by _id
for doc in doc_list:
doc_hash[bson.objectid.ObjectId(doc['_id'])] = doc
to_index = []
count = 0
while True:
try:
for doc in to_update:
if doc['_id'] in doc_hash:
del doc_hash[doc['_id']]
to_index.append(doc)
break
except (pymongo.errors.OperationFailure,
pymongo.errors.AutoReconnect):
count += 1
if count > 60:
sys.exit(1)
time.sleep(1)
#delete the inconsistent documents
for doc in doc_hash.values():
self.doc_manager.remove(doc)
#insert the ones from mongo
for doc in to_index:
doc['_ts'] = util.bson_ts_to_long(rollback_cutoff_ts)
doc['ns'] = namespace
try:
self.doc_manager.upsert(doc)
except SystemError:
logging.error("Unable to insert %s" % (doc))
return rollback_cutoff_ts
def test_rollback(self):
"""Test rollback in oplog_manager. Assertion failure if it doesn't pass
We force a rollback by inserting a doc, killing the primary,
inserting another doc, killing the new primary, and then restarting
both.
"""
os.system('rm config.txt; touch config.txt')
start_cluster()
test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog()
solr = DocManager()
test_oplog.doc_manager = solr
solr._delete() # equivalent to solr.delete(q='*: *')
obj1 = ObjectId('4ff74db3f646462b38000001')
mongos['test']['test'].remove({})
mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'},
safe=1)
while (mongos['test']['test'].find().count() != 1):
time.sleep(1)
cutoff_ts = test_oplog.get_last_oplog_timestamp()
obj2 = ObjectId('4ff74db3f646462b38000002')
first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts),
'ns': 'test.test',
'_id': obj1}
#try kill one, try restarting
killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
while admin.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
current_conn = mongos['test']['test']
current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1)
break
except:
count += 1
if count > 60:
string = 'Call to insert doc failed too many times'
logging.error(string)
sys.exit(1)
time.sleep(1)
continue
while (mongos['test']['test'].find().count() != 2):
print(mongos['test']['test'].find().count())
time.sleep(1)
killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY'])
startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
#wait for master to be established
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
#wait for secondary to be established
admin = new_primary_conn['admin']
while admin.command("replSetGetStatus")['myState'] != 2:
time.sleep(1)
while retry_until_ok(mongos['test']['test'].find().count) != 1:
time.sleep(1)
self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY'])
self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY'])
last_ts = test_oplog.get_last_oplog_timestamp()
second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts),
'ns': 'test.test', '_id': obj2}
test_oplog.doc_manager.upsert(first_doc)
test_oplog.doc_manager.upsert(second_doc)
test_oplog.rollback()
test_oplog.doc_manager.commit()
results = solr._search()
assert(len(results) == 1)
results_doc = results[0]
self.assertEqual(results_doc['name'], 'paulie')
self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
#test_oplog.join()
print("PASSED TEST ROLLBACK")
def dump_collection(self):
"""Dumps collection into the target system.
This method is called when we're initializing the cursor and have no
configs i.e. when we're starting for the first time.
"""
dump_set = self.namespace_set
#no namespaces specified
if not self.namespace_set:
db_list = self.main_connection.database_names()
for db in db_list:
if db == "config" or db == "local":
continue
coll_list = self.main_connection[db].collection_names()
for coll in coll_list:
if coll.startswith("system"):
continue
namespace = str(db) + "." + str(coll)
dump_set.append(namespace)
long_ts = None
for namespace in dump_set:
db, coll = namespace.split('.', 1)
target_coll = self.main_connection[db][coll]
cursor = util.retry_until_ok(target_coll.find)
cursor = cursor.sort('$natural', pymongo.DESCENDING)
oplog_cursor = util.retry_until_ok(self.oplog.find)
oplog_cursor = oplog_cursor.sort('$natural', pymongo.DESCENDING)
for entry in oplog_cursor:
if entry['op'] != 'i':
continue
#The 'o' field represents the document
search_doc = entry['o']
cursor.rewind()
for doc in cursor:
if search_doc == doc:
long_ts = util.bson_ts_to_long(entry['ts'])
break
if long_ts:
break
cursor.rewind()
try:
for doc in cursor:
doc['ns'] = namespace
doc['_ts'] = long_ts
self.doc_manager.upsert(doc)
except (pymongo.errors.AutoReconnect,
pymongo.errors.OperationFailure):
err_msg = "OplogManager: Failed during dump collection"
effect = "cannot recover!"
logging.error('%s %s %s' % (err_msg, effect, self.oplog))
self.running = False
return
if long_ts:
long_ts = util.long_to_bson_ts(long_ts)
else: # Implies that we are just initiating the set
long_ts = self.get_last_oplog_timestamp()
return long_ts
def test_rollback(self):
"""Test rollback in oplog_manager. Assertion failure if it doesn't pass
We force a rollback by inserting a doc, killing the primary,
inserting another doc, killing the new primary, and then restarting
both.
"""
os.system('rm config.txt; touch config.txt')
start_cluster()
test_oplog, primary_conn, mongos, oplog_coll = self.get_new_oplog()
solr = DocManager()
test_oplog.doc_manager = solr
solr._delete() # equivalent to solr.delete(q='*: *')
obj1 = ObjectId('4ff74db3f646462b38000001')
mongos['test']['test'].remove({})
mongos['test']['test'].insert({'_id': obj1, 'name': 'paulie'}, safe=1)
while (mongos['test']['test'].find().count() != 1):
time.sleep(1)
cutoff_ts = test_oplog.get_last_oplog_timestamp()
obj2 = ObjectId('4ff74db3f646462b38000002')
first_doc = {
'name': 'paulie',
'_ts': bson_ts_to_long(cutoff_ts),
'ns': 'test.test',
'_id': obj1
}
#try kill one, try restarting
killMongoProc(primary_conn.host, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection('localhost', int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
while admin.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
current_conn = mongos['test']['test']
current_conn.insert({'_id': obj2, 'name': 'paul'}, safe=1)
break
except:
count += 1
if count > 60:
string = 'Call to insert doc failed too many times'
logging.error(string)
sys.exit(1)
time.sleep(1)
continue
while (mongos['test']['test'].find().count() != 2):
print(mongos['test']['test'].find().count())
time.sleep(1)
killMongoProc(primary_conn.host, PORTS_ONE['SECONDARY'])
startMongoProc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
#wait for master to be established
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
startMongoProc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
#wait for secondary to be established
admin = new_primary_conn['admin']
while admin.command("replSetGetStatus")['myState'] != 2:
time.sleep(1)
while retry_until_ok(mongos['test']['test'].find().count) != 1:
time.sleep(1)
self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY'])
self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY'])
last_ts = test_oplog.get_last_oplog_timestamp()
second_doc = {
'name': 'paul',
'_ts': bson_ts_to_long(last_ts),
'ns': 'test.test',
'_id': obj2
}
test_oplog.doc_manager.upsert(first_doc)
test_oplog.doc_manager.upsert(second_doc)
test_oplog.rollback()
test_oplog.doc_manager.commit()
results = solr._search()
assert (len(results) == 1)
results_doc = results[0]
self.assertEqual(results_doc['name'], 'paulie')
self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
#test_oplog.join()
print("PASSED TEST ROLLBACK")
def rollback(self):
"""Rollback target system to consistent state.
The strategy is to find the latest timestamp in the target system and
the largest timestamp in the oplog less than the latest target system
timestamp. This defines the rollback window and we just roll these
back until the oplog and target system are in consistent states.
"""
self.doc_manager.commit()
last_inserted_doc = self.doc_manager.get_last_doc()
if last_inserted_doc is None:
return None
target_ts = util.long_to_bson_ts(last_inserted_doc['_ts'])
last_oplog_entry = self.oplog.find_one({'ts': {
'$lte': target_ts
}},
sort=[('$natural',
pymongo.DESCENDING)])
if last_oplog_entry is None:
return None
rollback_cutoff_ts = last_oplog_entry['ts']
start_ts = util.bson_ts_to_long(rollback_cutoff_ts)
end_ts = last_inserted_doc['_ts']
rollback_set = {} # this is a dictionary of ns:list of docs
for doc in self.doc_manager.search(start_ts, end_ts):
if doc['ns'] in rollback_set:
rollback_set[doc['ns']].append(doc)
else:
rollback_set[doc['ns']] = [doc]
for namespace, doc_list in rollback_set.items():
database, coll = namespace.split('.', 1)
obj_id = bson.objectid.ObjectId
bson_obj_id_list = [obj_id(doc['_id']) for doc in doc_list]
to_update = util.retry_until_ok(
self.main_connection[database][coll].find,
{'_id': {
'$in': bson_obj_id_list
}})
#doc list are docs in target system, to_update are docs in mongo
doc_hash = {} # hash by _id
for doc in doc_list:
doc_hash[bson.objectid.ObjectId(doc['_id'])] = doc
to_index = []
count = 0
while True:
try:
for doc in to_update:
if doc['_id'] in doc_hash:
del doc_hash[doc['_id']]
to_index.append(doc)
break
except (pymongo.errors.OperationFailure,
pymongo.errors.AutoReconnect):
count += 1
if count > 60:
sys.exit(1)
time.sleep(1)
#delete the inconsistent documents
for doc in doc_hash.values():
self.doc_manager.remove(doc)
#insert the ones from mongo
for doc in to_index:
doc['_ts'] = util.bson_ts_to_long(rollback_cutoff_ts)
doc['ns'] = namespace
try:
self.doc_manager.upsert(doc)
except SystemError:
logging.error("Unable to insert %s" % (doc))
return rollback_cutoff_ts
