def test_get_oplog_cursor(self):
"""Test the get_oplog_cursor method"""
# Trivial case: timestamp is None
self.assertEqual(self.opman1.get_oplog_cursor(None), None)
# earliest entry is after given timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.mongos_conn["test"]["mcsharded"].insert(doc)
self.assertEqual(self.opman1.get_oplog_cursor(bson.Timestamp(1, 0)),
None)
# earliest entry is the only one at/after timestamp
latest_timestamp = self.opman1.get_last_oplog_timestamp()
cursor = self.opman1.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = cursor[0]['o']['_id']
retrieved = self.mongos_conn.test.mcsharded.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
for i in range(2, 2002):
self.mongos_conn["test"]["mcsharded"].insert({"i": i})
oplog1 = self.shard1_conn["local"]["oplog.rs"].find(
sort=[("ts", pymongo.ASCENDING)])
oplog2 = self.shard2_conn["local"]["oplog.rs"].find(
sort=[("ts", pymongo.ASCENDING)])
# oplogs should have records for inserts performed, plus
# various other messages
oplog1_count = oplog1.count()
oplog2_count = oplog2.count()
self.assertGreaterEqual(oplog1_count, 998)
self.assertGreaterEqual(oplog2_count, 1002)
pivot1 = oplog1.skip(400).limit(1)[0]
pivot2 = oplog2.skip(400).limit(1)[0]
cursor1 = self.opman1.get_oplog_cursor(pivot1["ts"])
cursor2 = self.opman2.get_oplog_cursor(pivot2["ts"])
self.assertEqual(cursor1.count(), oplog1_count - 400)
self.assertEqual(cursor2.count(), oplog2_count - 400)
# get_oplog_cursor fast-forwards *one doc beyond* the given timestamp
doc1 = self.mongos_conn["test"]["mcsharded"].find_one(
{"_id": next(cursor1)["o"]["_id"]})
doc2 = self.mongos_conn["test"]["mcsharded"].find_one(
{"_id": next(cursor2)["o"]["_id"]})
piv1id = pivot1['o']['_id']
piv2id = pivot2['o']['_id']
retrieved1 = self.mongos_conn.test.mcsharded.find_one(piv1id)
retrieved2 = self.mongos_conn.test.mcsharded.find_one(piv2id)
self.assertEqual(doc1["i"], retrieved1["i"] + 1)
self.assertEqual(doc2["i"], retrieved2["i"] + 1)
def test_tail_with_messages(self):
op_timestamp = bson.Timestamp(long(time.time()), 0)
#some oplog message I copied from Mongo
op_message = {
"ts" : op_timestamp,
"h" : -2429474310205918006,
"op" : "u",
"ns" : "foodb.barcol",
"o2" : {
"_id" : "51d2daa81fa97fc9611102cf"
},
"o" : {
"$set" : {
"bar" : "baz"
}
}
}
self.cursor.sort.return_value = [op_message]
self.trigger.register("foodb.barcol", "u", self.callback_func)
new_checkpoint = self.trigger._tail_oplog(self.default_checkpoint)
self._assert_calls(self.default_checkpoint)
self.callback_func.assert_called_with(**op_message)
self.assertEquals(op_timestamp, new_checkpoint)
def timestamp(self, *args, **kwargs):
'''Create a bson.Timestamp. Factored into a method so we can delay
importing socket until gevent.monkey.patch_all() is called. You could
also insert another timestamp function to set options if you so desire.
'''
import bson
return bson.Timestamp(*args, **kwargs)
def test_get_oplog_cursor(self):
'''Test the get_oplog_cursor method'''
# timestamp is None - all oplog entries are returned.
cursor = self.opman.get_oplog_cursor(None)
self.assertEqual(cursor.count(),
self.primary_conn["local"]["oplog.rs"].count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.primary_conn["test"]["test"].insert(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.test.test.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["test"]["test"].insert({"i": i}
for i in range(2, 1002))
oplog_cursor = self.oplog_coll.find(sort=[("ts", pymongo.ASCENDING)])
# startup + insert + 1000 inserts
self.assertEqual(oplog_cursor.count(), 2 + 1000)
pivot = oplog_cursor.skip(400).limit(1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 2 + 1000 - 400)
def test_get_oplog_cursor(self):
"""Test the get_oplog_cursor method"""
# timestamp is None - all oplog entries excluding no-ops are returned.
cursor = self.opman.get_oplog_cursor(None)
self.assertEqual(cursor.count(),
self.primary_conn["local"]["oplog.rs"].find(
{'op': {'$ne': 'n'}}).count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.primary_conn["test"]["test"].insert_one(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.test.test.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["test"]["test"].insert_many(
[{"i": i} for i in range(2, 1002)])
oplog_cursor = self.oplog_coll.find(
{'op': {'$ne': 'n'},
'ns': {'$not': re.compile(r'\.(system|\$cmd)')}},
sort=[("ts", pymongo.ASCENDING)]
)
# initial insert + 1000 more inserts
self.assertEqual(oplog_cursor.count(), 1 + 1000)
pivot = oplog_cursor.skip(400).limit(-1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 1 + 1000 - 400)
def _set_and_get_checkpoint(self):
#is there a command in Mongo to do this in one shot: if the document doesn't exist just create it?
checkpoint_document = self._checkpoint.find_one(self.query_id) or {}
if not checkpoint_document:
self._checkpoint.save(self.query_id)
return checkpoint_document.get("checkpoint",
bson.Timestamp(long(time.time()), 0))
def main():
ts = None
fromhost = 'localhost:33000'
tohost = 'localhost:55000'
verbose = 0
for arg in sys.argv[1:]:
if arg == '--verbose':
verbose += 1
continue
match = re.match("--ts=(.*):(.*)", arg)
if match:
ts = bson.Timestamp(int(match.group(1)), int(match.group(2)))
if verbose: print("start after", ts)
continue
match = re.match("--fromhost=(.*)", arg)
if match:
fromhost = match.group(1)
continue
match = re.match("--tohost=(.*)", arg)
if match:
tohost = match.group(1)
continue
# connect to fromhost and tohost
try:
fromv = fromhost.split(':')
fromc = MongoClient(fromv[0], int(fromv[1]))
except:
print(fromv, sys.exc_info())
return 1
try:
tov = tohost.split(':')
toc = MongoClient(tov[0], int(tov[1]))
except:
print(tov, sys.exc_info())
return 1
# run a tailable cursor over the from host connection's oplog from a point in time
# and replay each oplog entry on the to host connection
db = fromc.local
oplog = db.oplog.rs
while 1:
if ts is None:
qs = {}
else:
qs = {'ts': {'$gt': ts}}
if verbose: print(qs)
c = oplog.find(qs, tailable=True, await_data=True)
if verbose: print(c)
if c.count() == 0:
time.sleep(1)
else:
for oploge in c:
if verbose: print(oploge)
op = oploge['op']
this_ts = oploge['ts']
replay(toc, op, oploge, verbose)
ts = this_ts
return 0
def __init__(self, cli):
self._cli = cli
self._coll = cli.local['oplog.rs']
last_msg = self.last()
if last_msg:
self._position = last_msg['ts']
else:
self._position = bson.Timestamp(0, 1)
def test_get_oplog_cursor(self):
"""Test the get_oplog_cursor method"""
# Put something in the dbs
self.init_dbs()
# timestamp is None - all oplog entries excluding no-ops are returned.
# wildcard include case no impact the result
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
got_cursor = self.opman.get_oplog_cursor(None)
oplog_cursor = self.oplog_coll.find({"op": {"$ne": "n"}})
self.assertNotEqual(got_cursor, None)
self.assertEqual(got_cursor.count(), oplog_cursor.count())
# wildcard exclude case no impact the result
self.reset_opman([], ["includedb2.excludecol2", "excludedb3.*"], {})
got_cursor = self.opman.get_oplog_cursor(None)
oplog_cursor = self.oplog_coll.find({"op": {"$ne": "n"}})
self.assertNotEqual(got_cursor, None)
self.assertEqual(got_cursor.count(), oplog_cursor.count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "idb1col1": 1}
self.primary_conn["includedb1"]["includecol1"].insert_one(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)["o"]["_id"]
retrieved = self.primary_conn.includedb1.includecol1.find_one(
next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["includedb1"]["includecol1"].insert_many([{
"idb1col1":
i
} for i in range(2, 1002)])
oplog_cursor = self.oplog_coll.find(
{
"op": {
"$ne": "n"
},
"ns": {
"$not": re.compile(r"\.(system|\$cmd)")
}
},
sort=[("ts", pymongo.ASCENDING)],
)
# initial insert + 1000 more inserts
self.assertEqual(oplog_cursor.count(), 11 + 1000)
pivot = oplog_cursor.skip(400).limit(-1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 11 + 1000 - 400)
def test_get_oplog_cursor(self):
"""Test the get_oplog_cursor method"""
# timestamp = None
cursor1 = self.opman1.get_oplog_cursor(None)
oplog1 = self.shard1_conn["local"]["oplog.rs"].find({
'op': {
'$ne': 'n'
},
'ns': {
'$not': re.compile(r'\.system')
}
})
self.assertEqual(list(cursor1), list(oplog1))
cursor2 = self.opman2.get_oplog_cursor(None)
oplog2 = self.shard2_conn["local"]["oplog.rs"].find({
'op': {
'$ne': 'n'
},
'ns': {
'$not': re.compile(r'\.system')
}
})
self.assertEqual(list(cursor2), list(oplog2))
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.mongos_conn["test"]["mcsharded"].insert_one(doc)
latest_timestamp = self.opman1.get_last_oplog_timestamp()
cursor = self.opman1.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = cursor[0]['o']['_id']
retrieved = self.mongos_conn.test.mcsharded.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
for i in range(2, 2002):
self.mongos_conn["test"]["mcsharded"].insert_one({"i": i})
oplog1 = self.shard1_conn["local"]["oplog.rs"].find(
sort=[("ts", pymongo.ASCENDING)])
oplog2 = self.shard2_conn["local"]["oplog.rs"].find(
sort=[("ts", pymongo.ASCENDING)])
# oplogs should have records for inserts performed, plus
# various other messages
oplog1_count = oplog1.count()
oplog2_count = oplog2.count()
self.assertGreaterEqual(oplog1_count, 998)
self.assertGreaterEqual(oplog2_count, 1002)
pivot1 = oplog1.skip(400).limit(-1)[0]
pivot2 = oplog2.skip(400).limit(-1)[0]
cursor1 = self.opman1.get_oplog_cursor(pivot1["ts"])
cursor2 = self.opman2.get_oplog_cursor(pivot2["ts"])
self.assertEqual(cursor1.count(), oplog1_count - 400)
self.assertEqual(cursor2.count(), oplog2_count - 400)
def test_bson_types(self):
i = {
"a": bson.ObjectId("55153a8014829a865bbf700d"),
"b": bson.datetime.datetime.now(),
"c": bson.Timestamp(0, 0),
}
expected = {"a": "OID", "b": "DATETIME", "c": "TIMESTAMP"}
s = extract.extract_schema_from_document(i, {})
for key, value in expected.items():
types = list(s[key]["types"].keys())
self.assertListEqual([value], types)
def get_last_optime(client, fixturelib):
"""Get the latest optime.
This function is derived from _getLastOpTime() in ReplSetTest.
"""
repl_set_status = client.admin.command({"replSetGetStatus": 1})
conn_status = [m for m in repl_set_status["members"] if "self" in m][0]
optime = conn_status["optime"]
optime_is_empty = False
if isinstance(optime, bson.Timestamp): # PV0
optime_is_empty = (optime == bson.Timestamp(0, 0))
else: # PV1
optime_is_empty = (optime["ts"] == bson.Timestamp(0, 0) and optime["t"] == -1)
if optime_is_empty:
raise fixturelib.ServerFailure(
"Uninitialized opTime being reported by {addr[0]}:{addr[1]}: {repl_set_status}".format(
addr=client.address, repl_set_status=repl_set_status))
return optime
def test_get_oplog_cursor(self):
'''Test the get_oplog_cursor method'''
# Trivial case: timestamp is None
self.assertEqual(self.opman.get_oplog_cursor(None), None)
# earliest entry is after given timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.primary_conn["test"]["test"].insert(doc)
self.assertEqual(self.opman.get_oplog_cursor(bson.Timestamp(1, 0)),
None)
# earliest entry is the only one at/after timestamp
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.test.test.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["test"]["test"].insert({"i": i}
for i in range(2, 1002))
oplog_cursor = self.oplog_coll.find(sort=[("ts", pymongo.ASCENDING)])
# startup + insert + 1000 inserts
self.assertEqual(oplog_cursor.count(), 2 + 1000)
pivot = oplog_cursor.skip(400).limit(1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 2 + 1000 - 400)
# get_oplog_cursor fast-forwards *one doc beyond* the given timestamp
doc = self.primary_conn["test"]["test"].find_one(
{"_id": next(goc_cursor)["o"]["_id"]})
retrieved = self.primary_conn.test.test.find_one(pivot['o']['_id'])
self.assertEqual(doc["i"], retrieved["i"] + 1)
def test_tail_with_nonmatching_operation(self):
op_timestamp = bson.Timestamp(long(time.time()), 0)
op_message = {
"ts" : op_timestamp,
"op" : "u", #this is an update message
"ns" : "foodb.barcol"
}
self.cursor.sort.return_value = [op_message]
self.trigger.register("foodb.barcol", "i", self.callback_func) #only registering insert ("i") messages
new_checkpoint = self.trigger._tail_oplog(self.default_checkpoint)
self._assert_calls(self.default_checkpoint)
self.assertFalse(self.callback_func.called)
self.assertEquals(op_timestamp, new_checkpoint)
def test_tail_with_nonmatching_namespace(self):
op_timestamp = bson.Timestamp(long(time.time()), 0)
op_message = {
"ts" : op_timestamp,
"op" : "u",
"ns" : "foodb.barcol"
}
self.cursor.sort.return_value = [op_message]
self.trigger.register("adifferentdb.adifferentcol", "u", self.callback_func)
new_checkpoint = self.trigger._tail_oplog(self.default_checkpoint)
self._assert_calls(self.default_checkpoint)
self.assertFalse(self.callback_func.called)
self.assertEquals(op_timestamp, new_checkpoint)
def __init__(self):
self.ops_retrieved = 0
self.inserts = 0
self.insert_warnings = 0
self.deletes = 0
self.delete_warnings = 0
self.updates = 0
self.update_warnings = 0
self.last_ts = bson.Timestamp(int(time.time()), 0)
self.sleeps = 0
self.exceptions = 0
self.retries = 0
self.paused = False
self.pending_ids = set()
def replicate(self, master_name, checkpoint=None):
'''Actual replication loop for replicating off of master_uri'''
master_repl_config = self._config[master_name]
master_info = self._topology[master_name]
master_id = master_repl_config['_id']
conn = self.connect(master_info['uri'])
if checkpoint is None:
checkpoint = master_repl_config.get('checkpoint')
if checkpoint is None:
# By default, start replicating as of NOW
checkpoint = bson.Timestamp(long(time.time()), 0)
triggers = Triggers(conn, checkpoint)
for repl in master_repl_config['replication']:
triggers.register(
repl['src'], repl['ops'],
self._replicate_to_trigger(master_id, repl['dst']))
for checkpoint in triggers.run():
master_repl_config['checkpoint'] = checkpoint
def sync(args):
last_ts = load_ckpt(args.ckpt)
ts = last_ts if last_ts is not None else int(args.ts)
ts = bson.Timestamp(ts, 0)
client = pymongo.MongoClient(args.src_url)
oplog = client.local.oplog.rs
db, tbl = args.dst_ns.split('.')
client = pymongo.MongoClient(args.dst_url)
table = client[db][tbl]
buffers = []
flush_ts = time.time()
while True:
# For a regular capped collection CursorType.TAILABLE_AWAIT is the
# only option required to create a tailable cursor. When querying the
# oplog the oplog_replay option enables an optimization to quickly
# find the 'ts' value we're looking for. The oplog_replay option
# can only be used when querying the oplog.
cursor = oplog.find({
'ts': {
'$gt': ts
},
'ns': args.src_ns
},
cursor_type=pymongo.CursorType.TAILABLE_AWAIT,
oplog_replay=True)
while cursor.alive:
for doc in cursor:
ts = doc['ts']
buffers.append(doc)
now = time.time()
if (now - flush_ts) > 2 or len(buffers) >= args.bufsize:
flush_buffers(table, buffers, args.ordered, args.dry_run)
write_ckpt(args.ckpt, ts)
buffers = []
flush_ts = now
# We end up here if the find() returned no documents or if the
# tailable cursor timed out (no new documents were added to the
# collection for more than 1 second).
time.sleep(1)
def test_collection_rename_on_create_cmd(self):
"""
Starting in MongoDB 3.2, a create collection is required
before insert operations (apparently). Ensure that a renamed
command in a create collection is renamed.
"""
op = {
'h': -4317026186822365585,
't': 1,
'ns': 'newdb.$cmd',
'v': 2,
'o': {'create': 'coll_1'},
'ts': bson.Timestamp(1470940276, 1),
'op': 'c',
}
ren = oplog.Renamer.from_specs("newdb.coll_1=newdb.coll_2")
ren(op)
assert op['o']['create'] == 'coll_2'
def await (self, spec=None):
'''Await the very next message on the oplog satisfying the spec'''
if spec is None:
spec = {}
last = self.last(spec)
if last is None:
return # Can't await unless there is an existing message satisfying spec
await_spec = dict(spec)
last_ts = last['ts']
await_spec['ts'] = {
'$gt': bson.Timestamp(last_ts.time, last_ts.inc - 1)
}
curs = self._coll.find(await_spec, tailable=True, await_data=True)
curs = curs.hint([('$natural', 1)])
curs = curs.add_option(_QUERY_OPTIONS['oplog_replay'])
curs.next() # should always find 1 element
try:
return curs.next()
except StopIteration:
return None
def test_rename_index_op_ns(self):
"""
As an index operation references namespaces,
when performing a rename operation, it's also important
to rename the ns in the op itself.
"""
op = {
'ts': bson.Timestamp(1446495808, 3),
'ns': 'airportlocker.system.indexes',
'op': 'i',
'o': {
'ns': 'airportlocker.luggage.chunks',
'key': {'files_id': 1, 'n': 1},
'name': 'files_id_1_n_1',
'unique': True,
},
}
ren = oplog.Renamer.from_specs("airportlocker=airportlocker-us")
ren(op)
assert op['ns'] == 'airportlocker-us.system.indexes'
assert op['o']['ns'] == 'airportlocker-us.luggage.chunks'
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
self.opman.collection_dump = False
self.assertTrue(
all(doc['op'] == 'n' for doc in self.opman.init_cursor()[0]))
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman.collection_dump = True
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor, None)
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman.oplog = self.primary_conn['local']['oplog.rs']
collection = self.primary_conn["test"]["test"]
collection.insert_one({"i": 1})
collection.delete_one({"i": 1})
time.sleep(3)
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, last_ts)
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts)
# No last checkpoint, no collection dump, something in oplog
self.opman.oplog_progress = LockingDict()
self.opman.collection_dump = False
collection.insert_one({"i": 2})
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
for i in range(cursor_len - 1):
next(cursor)
self.assertEqual(next(cursor)['o']['i'], 2)
self.assertEqual(self.opman.checkpoint, last_ts)
# Last checkpoint exists
progress = LockingDict()
self.opman.oplog_progress = progress
for i in range(1000):
collection.insert_one({"i": i + 500})
entry = list(self.primary_conn["local"]["oplog.rs"].find(skip=200,
limit=-2))
progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"]
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(next(cursor)["ts"], entry[1]["ts"])
self.assertEqual(self.opman.checkpoint, entry[0]["ts"])
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)],
entry[0]["ts"])
# Last checkpoint is behind
progress = LockingDict()
progress.get_dict()[str(self.opman.oplog)] = bson.Timestamp(1, 0)
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman.checkpoint)
def convertTime(Time):
tm = bson.Timestamp(Time, 1)
return tm.as_datetime()
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman1.oplog = self.shard1_conn["test"]["emptycollection"]
self.opman2.oplog = self.shard2_conn["test"]["emptycollection"]
self.opman1.collection_dump = False
self.opman2.collection_dump = False
self.assertTrue(
all(doc['op'] == 'n' for doc in self.opman1.init_cursor()[0]))
self.assertEqual(self.opman1.checkpoint, None)
self.assertTrue(
all(doc['op'] == 'n' for doc in self.opman2.init_cursor()[0]))
self.assertEqual(self.opman2.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman1.collection_dump = self.opman2.collection_dump = True
cursor, cursor_len = self.opman1.init_cursor()
self.assertEqual(cursor, None)
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman1.checkpoint, None)
cursor, cursor_len = self.opman2.init_cursor()
self.assertEqual(cursor, None)
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman2.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman1.oplog = self.shard1_conn["local"]["oplog.rs"]
self.opman2.oplog = self.shard2_conn["local"]["oplog.rs"]
oplog_startup_ts = self.opman2.get_last_oplog_timestamp()
collection = self.mongos_conn["test"]["mcsharded"]
collection.insert({"i": 1})
collection.remove({"i": 1})
time.sleep(3)
last_ts1 = self.opman1.get_last_oplog_timestamp()
cursor, cursor_len = self.opman1.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman1.checkpoint, last_ts1)
with self.opman1.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1)
# init_cursor should point to startup message in shard2 oplog
cursor, cursor_len = self.opman2.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman2.checkpoint, oplog_startup_ts)
# No last checkpoint, no collection dump, stuff in oplog
progress = LockingDict()
self.opman1.oplog_progress = self.opman2.oplog_progress = progress
self.opman1.collection_dump = self.opman2.collection_dump = False
collection.insert({"i": 1200})
last_ts2 = self.opman2.get_last_oplog_timestamp()
self.opman1.init_cursor()
self.assertEqual(self.opman1.checkpoint, last_ts1)
with self.opman1.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman1.oplog)], last_ts1)
cursor, cursor_len = self.opman2.init_cursor()
for i in range(cursor_len - 1):
next(cursor)
self.assertEqual(next(cursor)["o"]["i"], 1200)
self.assertEqual(self.opman2.checkpoint, last_ts2)
with self.opman2.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman2.oplog)], last_ts2)
# Last checkpoint exists
progress = LockingDict()
self.opman1.oplog_progress = self.opman2.oplog_progress = progress
for i in range(1000):
collection.insert({"i": i + 500})
entry1 = list(self.shard1_conn["local"]["oplog.rs"].find(skip=200,
limit=2))
entry2 = list(self.shard2_conn["local"]["oplog.rs"].find(skip=200,
limit=2))
progress.get_dict()[str(self.opman1.oplog)] = entry1[0]["ts"]
progress.get_dict()[str(self.opman2.oplog)] = entry2[0]["ts"]
self.opman1.oplog_progress = self.opman2.oplog_progress = progress
self.opman1.checkpoint = self.opman2.checkpoint = None
cursor1, cursor_len1 = self.opman1.init_cursor()
cursor2, cursor_len2 = self.opman2.init_cursor()
self.assertEqual(entry1[1]["ts"], next(cursor1)["ts"])
self.assertEqual(entry2[1]["ts"], next(cursor2)["ts"])
self.assertEqual(self.opman1.checkpoint, entry1[0]["ts"])
self.assertEqual(self.opman2.checkpoint, entry2[0]["ts"])
with self.opman1.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman1.oplog)],
entry1[0]["ts"])
with self.opman2.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman2.oplog)],
entry2[0]["ts"])
# Last checkpoint is behind
progress = LockingDict()
progress.get_dict()[str(self.opman1.oplog)] = bson.Timestamp(1, 0)
progress.get_dict()[str(self.opman2.oplog)] = bson.Timestamp(1, 0)
self.opman1.oplog_progress = self.opman2.oplog_progress = progress
self.opman1.checkpoint = self.opman2.checkpoint = None
cursor, cursor_len = self.opman1.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman1.checkpoint)
cursor, cursor_len = self.opman2.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman2.checkpoint)
def _check_invariants_as_standalone(self, secondary):
# We remove the --replSet option in order to start the node as a standalone.
replset_name = secondary.mongod_options.pop("replSet")
try:
secondary.setup()
secondary.await_ready()
client = secondary.mongo_client()
minvalid_doc = client.local["replset.minvalid"].find_one()
oplog_truncate_after_doc = client.local[
"replset.oplogTruncateAfterPoint"].find_one()
checkpoint_timestamp_doc = client.local[
"replset.checkpointTimestamp"].find_one()
latest_oplog_doc = client.local["oplog.rs"].find_one(
sort=[("$natural", pymongo.DESCENDING)])
null_ts = bson.Timestamp(0, 0)
# The oplog could be empty during initial sync. If so, we default it to null.
latest_oplog_entry_ts = null_ts
if latest_oplog_doc is not None:
latest_oplog_entry_ts = latest_oplog_doc.get("ts")
if latest_oplog_entry_ts is None:
raise errors.ServerFailure(
"Latest oplog entry had no 'ts' field: {}".format(
latest_oplog_doc))
# The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default
# it to null.
oplog_truncate_after_ts = null_ts
if oplog_truncate_after_doc is not None:
oplog_truncate_after_ts = oplog_truncate_after_doc.get(
"oplogTruncateAfterPoint", null_ts)
# The "checkpointTimestamp" document may not exist at startup. If so, we default
# it to null.
checkpoint_timestamp = null_ts
if checkpoint_timestamp_doc is not None:
checkpoint_timestamp = checkpoint_timestamp_doc.get(
"checkpointTimestamp")
if checkpoint_timestamp is None:
raise errors.ServerFailure(
"Checkpoint timestamp document had no 'checkpointTimestamp'"
"field: {}".format(checkpoint_timestamp_doc))
# checkpointTimestamp <= top of oplog
# If the oplog is empty, the checkpoint timestamp should also be null.
if not checkpoint_timestamp <= latest_oplog_entry_ts:
raise errors.ServerFailure(
"The condition checkpointTimestamp <= top of oplog ({} <= {}) doesn't hold:"
" checkpointTimestamp document={}, latest oplog entry={}".
format(checkpoint_timestamp, latest_oplog_entry_ts,
checkpoint_timestamp_doc, latest_oplog_doc))
if minvalid_doc is not None:
applied_through_ts = minvalid_doc.get("begin",
{}).get("ts", null_ts)
minvalid_ts = minvalid_doc.get("ts", null_ts)
# The "appliedThrough" value should always equal the "checkpointTimestamp".
# The writes to "appliedThrough" are given the timestamp of the end of the batch,
# and batch boundaries are the only valid timestamps in which we could take
# checkpoints, so if you see a non-null applied through in a stable checkpoint it
# must be at the same timestamp as the checkpoint.
if (checkpoint_timestamp != null_ts
and applied_through_ts != null_ts
and (not checkpoint_timestamp == applied_through_ts)):
raise errors.ServerFailure(
"The condition checkpointTimestamp ({}) == appliedThrough ({})"
" doesn't hold: minValid document={},"
" checkpointTimestamp document={}, last oplog entry={}"
.format(checkpoint_timestamp, applied_through_ts,
minvalid_doc, checkpoint_timestamp_doc,
latest_oplog_doc))
if applied_through_ts == null_ts:
# We clear "appliedThrough" to represent having applied through the top of the
# oplog in PRIMARY state or immediately after "rollback via refetch".
# If we are using a storage engine that supports "recover to a checkpoint,"
# then we will have a "checkpointTimestamp" and we should use that as our
# "appliedThrough" (similarly to why we assert their equality above).
# If both are null, then we are in PRIMARY state on a storage engine that does
# not support "recover to a checkpoint" or in RECOVERING immediately after
# "rollback via refetch". Since we do not update "minValid" in PRIMARY state,
# we leave "appliedThrough" as null so that the invariants below hold, rather
# than substituting the latest oplog entry for the "appliedThrough" value.
applied_through_ts = checkpoint_timestamp
if minvalid_ts == null_ts:
# The server treats the "ts" field in the minValid document as missing when its
# value is the null timestamp.
minvalid_ts = applied_through_ts
if latest_oplog_entry_ts == null_ts:
# If the oplog is empty, we treat the "minValid" as the latest oplog entry.
latest_oplog_entry_ts = minvalid_ts
if oplog_truncate_after_ts == null_ts:
# The server treats the "oplogTruncateAfterPoint" field as missing when its
# value is the null timestamp. When it is null, the oplog is complete and
# should not be truncated, so it is effectively the top of the oplog.
oplog_truncate_after_ts = latest_oplog_entry_ts
# Check the ordering invariants before the secondary has reconciled the end of
# its oplog.
# The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of
# oplog entries before they are written to the oplog. Thus, it can be ahead
# of the top of the oplog before any oplog entries are written, and behind it
# after some are written. Thus, we cannot compare it to the top of the oplog.
# appliedThrough <= minValid
# appliedThrough represents the end of the previous batch, so it is always the
# earliest.
if not applied_through_ts <= minvalid_ts:
raise errors.ServerFailure(
"The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid"
" document={}, latest oplog entry={}".format(
applied_through_ts, minvalid_ts, minvalid_doc,
latest_oplog_doc))
# minValid <= oplogTruncateAfterPoint
# This is true because this hook is never run after a rollback. Thus, we only
# move "minValid" to the end of each batch after the batch is written to the oplog.
# We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from
# the end of the previous batch to the end of the current batch. Thus "minValid"
# must be less than or equal to the "oplogTruncateAfterPoint".
if not minvalid_ts <= oplog_truncate_after_ts:
raise errors.ServerFailure(
"The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't"
" hold: minValid document={}, oplogTruncateAfterPoint document={},"
" latest oplog entry={}".format(
minvalid_ts, oplog_truncate_after_ts, minvalid_doc,
oplog_truncate_after_doc, latest_oplog_doc))
# minvalid <= latest oplog entry
# "minValid" is set to the end of a batch after the batch is written to the oplog.
# Thus it is always less than or equal to the top of the oplog.
if not minvalid_ts <= latest_oplog_entry_ts:
raise errors.ServerFailure(
"The condition minValid <= top of oplog ({} <= {}) doesn't"
" hold: minValid document={}, latest oplog entry={}".
format(minvalid_ts, latest_oplog_entry_ts,
minvalid_doc, latest_oplog_doc))
try:
secondary.teardown()
except errors.ServerFailure:
raise errors.ServerFailure(
"{} did not exit cleanly after being started up as a standalone"
.format(secondary))
except pymongo.errors.OperationFailure as err:
self.logger.exception(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" the checkpointTimestamp document, or the latest oplog entry from the mongod on"
" port %d", secondary.port)
raise errors.ServerFailure(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" the checkpointTimestamp document, or the latest oplog entry from the mongod on"
" port {}: {}".format(secondary.port, err.args[0]))
finally:
# Set the secondary's options back to their original values.
secondary.mongod_options["replSet"] = replset_name
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman.oplog = self.primary_conn["includedb1"]["emptycollection"]
self.opman.collection_dump = False
self.assertTrue(
all(doc['op'] == 'n' for doc in self.opman.init_cursor()[0]))
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman.collection_dump = True
cursor, cursor_empty = self.opman.init_cursor()
self.assertEqual(cursor, None)
self.assertTrue(cursor_empty)
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman.oplog = self.primary_conn['local']['oplog.rs']
collection = self.primary_conn["includedb1"]["includecol1"]
collection.insert_one({"idb1col1": 1})
collection.delete_one({"idb1col1": 1})
time.sleep(3)
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_empty = self.opman.init_cursor()
self.assertFalse(cursor_empty)
self.assertEqual(self.opman.checkpoint, last_ts)
self.assertEqual(self.opman.read_last_checkpoint(), last_ts)
# No last checkpoint, no collection dump, something in oplog
# If collection dump is false the checkpoint should not be set
self.opman.checkpoint = None
self.opman.oplog_progress = LockingDict()
self.opman.collection_dump = False
collection.insert_one({"idb1col1": 2})
cursor, cursor_empty = self.opman.init_cursor()
for doc in cursor:
last_doc = doc
self.assertEqual(last_doc['o']['idb1col1'], 2)
self.assertIsNone(self.opman.checkpoint)
# Last checkpoint exists
collection.insert_many([{"idb1col1": i + 500} for i in range(1000)])
entry = list(self.primary_conn["local"]["oplog.rs"].find(skip=200,
limit=-2))
self.opman.update_checkpoint(entry[0]["ts"])
cursor, cursor_empty = self.opman.init_cursor()
self.assertEqual(next(cursor)["ts"], entry[1]["ts"])
self.assertEqual(self.opman.checkpoint, entry[0]["ts"])
self.assertEqual(self.opman.read_last_checkpoint(), entry[0]["ts"])
# Last checkpoint is behind
self.opman.update_checkpoint(bson.Timestamp(1, 0))
cursor, cursor_empty = self.opman.init_cursor()
self.assertTrue(cursor_empty)
self.assertEqual(cursor, None)
self.assertEqual(self.opman.checkpoint, bson.Timestamp(1, 0))
def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-branches
# We remove the --replSet option in order to start the node as a standalone.
replset_name = secondary.mongod_options.pop("replSet")
try:
secondary.setup()
secondary.await_ready()
client = secondary.mongo_client()
minvalid_doc = client.local["replset.minvalid"].find_one()
oplog_truncate_after_doc = client.local[
"replset.oplogTruncateAfterPoint"].find_one()
self.logger.info("minValid: {}, oTAP: {}".format(
minvalid_doc, oplog_truncate_after_doc))
latest_oplog_doc = client.local["oplog.rs"].find_one(
sort=[("$natural", pymongo.DESCENDING)])
null_ts = bson.Timestamp(0, 0)
# The oplog could be empty during initial sync. If so, we default it to null.
latest_oplog_entry_ts = null_ts
if latest_oplog_doc is not None:
latest_oplog_entry_ts = latest_oplog_doc.get("ts")
if latest_oplog_entry_ts is None:
raise errors.ServerFailure(
"Latest oplog entry had no 'ts' field: {}".format(
latest_oplog_doc))
# The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default
# it to null.
oplog_truncate_after_ts = null_ts
if oplog_truncate_after_doc is not None:
oplog_truncate_after_ts = oplog_truncate_after_doc.get(
"oplogTruncateAfterPoint", null_ts)
if minvalid_doc is not None:
applied_through_ts = minvalid_doc.get("begin",
{}).get("ts", null_ts)
minvalid_ts = minvalid_doc.get("ts", null_ts)
if minvalid_ts == null_ts:
# The server treats the "ts" field in the minValid document as missing when its
# value is the null timestamp.
minvalid_ts = applied_through_ts
if latest_oplog_entry_ts == null_ts:
# If the oplog is empty, we treat the "minValid" as the latest oplog entry.
latest_oplog_entry_ts = minvalid_ts
if oplog_truncate_after_ts == null_ts:
# The server treats the "oplogTruncateAfterPoint" field as missing when its
# value is the null timestamp. When it is null, the oplog is complete and
# should not be truncated, so it is effectively the top of the oplog.
oplog_truncate_after_ts = latest_oplog_entry_ts
# Check the ordering invariants before the secondary has reconciled the end of
# its oplog.
# The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of
# oplog entries before they are written to the oplog. Thus, it can be ahead
# of the top of the oplog before any oplog entries are written, and behind it
# after some are written. Thus, we cannot compare it to the top of the oplog.
# appliedThrough <= minValid
# appliedThrough represents the end of the previous batch, so it is always the
# earliest.
if not applied_through_ts <= minvalid_ts:
raise errors.ServerFailure(
"The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid"
" document={}, latest oplog entry={}".format(
applied_through_ts, minvalid_ts, minvalid_doc,
latest_oplog_doc))
# minValid <= oplogTruncateAfterPoint
# This is true because this hook is never run after a rollback. Thus, we only
# move "minValid" to the end of each batch after the batch is written to the oplog.
# We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from
# the end of the previous batch to the end of the current batch. Thus "minValid"
# must be less than or equal to the "oplogTruncateAfterPoint".
if not minvalid_ts <= oplog_truncate_after_ts:
raise errors.ServerFailure(
"The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't"
" hold: minValid document={}, oplogTruncateAfterPoint document={},"
" latest oplog entry={}".format(
minvalid_ts, oplog_truncate_after_ts, minvalid_doc,
oplog_truncate_after_doc, latest_oplog_doc))
# minvalid <= latest oplog entry
# "minValid" is set to the end of a batch after the batch is written to the oplog.
# Thus it is always less than or equal to the top of the oplog.
if not minvalid_ts <= latest_oplog_entry_ts:
raise errors.ServerFailure(
"The condition minValid <= top of oplog ({} <= {}) doesn't"
" hold: minValid document={}, latest oplog entry={}".
format(minvalid_ts, latest_oplog_entry_ts,
minvalid_doc, latest_oplog_doc))
try:
secondary.teardown()
except errors.ServerFailure:
raise errors.ServerFailure(
"{} did not exit cleanly after being started up as a standalone"
.format(secondary))
except pymongo.errors.OperationFailure as err:
self.logger.exception(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" or the latest oplog entry from the mongod on port %d",
secondary.port)
raise errors.ServerFailure(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" or the latest oplog entry from the mongod on"
" port {}: {}".format(secondary.port, err.args[0]))
finally:
# Set the secondary's options back to their original values.
secondary.mongod_options["replSet"] = replset_name
def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-locals
# pylint: disable=too-many-branches,too-many-statements
# We remove the --replSet option in order to start the node as a standalone.
replset_name = secondary.mongod_options.pop("replSet")
self.logger.info(
"Restarting the secondary on port %d as a standalone node with"
" its data files intact...", secondary.port)
try:
secondary.setup()
secondary.await_ready()
client = secondary.mongo_client()
minvalid_doc = client.local["replset.minvalid"].find_one()
oplog_truncate_after_doc = client.local["replset.oplogTruncateAfterPoint"].find_one()
recovery_timestamp_res = client.admin.command("replSetTest",
getLastStableRecoveryTimestamp=True)
latest_oplog_doc = client.local["oplog.rs"].find_one(sort=[("$natural",
pymongo.DESCENDING)])
self.logger.info("Checking invariants: minValid: {}, oplogTruncateAfterPoint: {},"
" stable recovery timestamp: {}, latest oplog doc: {}".format(
minvalid_doc, oplog_truncate_after_doc, recovery_timestamp_res,
latest_oplog_doc))
null_ts = bson.Timestamp(0, 0)
# We wait for a stable recovery timestamp at setup, so we must have an oplog.
latest_oplog_entry_ts = null_ts
if latest_oplog_doc is None:
raise errors.ServerFailure("No latest oplog entry")
latest_oplog_entry_ts = latest_oplog_doc.get("ts")
if latest_oplog_entry_ts is None:
raise errors.ServerFailure(
"Latest oplog entry had no 'ts' field: {}".format(latest_oplog_doc))
# The "oplogTruncateAfterPoint" document may not exist at startup. If so, we default
# it to null.
oplog_truncate_after_ts = null_ts
if oplog_truncate_after_doc is not None:
oplog_truncate_after_ts = oplog_truncate_after_doc.get(
"oplogTruncateAfterPoint", null_ts)
# The "lastStableRecoveryTimestamp" field is present if the storage engine supports
# "recover to a timestamp". If it's a null timestamp on a durable storage engine, that
# means we do not yet have a stable checkpoint timestamp and must be restarting at the
# top of the oplog. Since we wait for a stable recovery timestamp at test fixture setup,
# we should never encounter a null timestamp here.
recovery_timestamp = recovery_timestamp_res.get("lastStableRecoveryTimestamp")
if recovery_timestamp == null_ts:
raise errors.ServerFailure(
"Received null stable recovery timestamp {}".format(recovery_timestamp_res))
# On a storage engine that doesn't support "recover to a timestamp", we default to null.
if recovery_timestamp is None:
recovery_timestamp = null_ts
# last stable recovery timestamp <= top of oplog
if not recovery_timestamp <= latest_oplog_entry_ts:
raise errors.ServerFailure("The condition last stable recovery timestamp <= top"
" of oplog ({} <= {}) doesn't hold:"
" getLastStableRecoveryTimestamp result={},"
" latest oplog entry={}".format(
recovery_timestamp, latest_oplog_entry_ts,
recovery_timestamp_res, latest_oplog_doc))
if minvalid_doc is not None:
applied_through_ts = minvalid_doc.get("begin", {}).get("ts", null_ts)
minvalid_ts = minvalid_doc.get("ts", null_ts)
# The "appliedThrough" value should always equal the "last stable recovery
# timestamp", AKA the stable checkpoint for durable engines, on server restart.
#
# The written "appliedThrough" time is updated with the latest timestamp at the end
# of each batch application, and batch boundaries are the only valid stable
# timestamps on secondaries. Therefore, a non-null appliedThrough timestamp must
# equal the checkpoint timestamp, because any stable timestamp that the checkpoint
# could use includes an equal persisted appliedThrough timestamp.
if (recovery_timestamp != null_ts and applied_through_ts != null_ts
and (not recovery_timestamp == applied_through_ts)):
raise errors.ServerFailure(
"The condition last stable recovery timestamp ({}) == appliedThrough ({})"
" doesn't hold: minValid document={},"
" getLastStableRecoveryTimestamp result={}, last oplog entry={}".format(
recovery_timestamp, applied_through_ts, minvalid_doc,
recovery_timestamp_res, latest_oplog_doc))
if applied_through_ts == null_ts:
# We clear "appliedThrough" to represent having applied through the top of the
# oplog in PRIMARY state or immediately after "rollback via refetch".
# If we are using a storage engine that supports "recover to a timestamp,"
# then we will have a "last stable recovery timestamp" and we should use that
# as our "appliedThrough" (similarly to why we assert their equality above).
# If both are null, then we are in PRIMARY state on a storage engine that does
# not support "recover to a timestamp" or in RECOVERING immediately after
# "rollback via refetch". Since we do not update "minValid" in PRIMARY state,
# we leave "appliedThrough" as null so that the invariants below hold, rather
# than substituting the latest oplog entry for the "appliedThrough" value.
applied_through_ts = recovery_timestamp
if minvalid_ts == null_ts:
# The server treats the "ts" field in the minValid document as missing when its
# value is the null timestamp.
minvalid_ts = applied_through_ts
if latest_oplog_entry_ts == null_ts:
# If the oplog is empty, we treat the "minValid" as the latest oplog entry.
latest_oplog_entry_ts = minvalid_ts
if oplog_truncate_after_ts == null_ts:
# The server treats the "oplogTruncateAfterPoint" field as missing when its
# value is the null timestamp. When it is null, the oplog is complete and
# should not be truncated, so it is effectively the top of the oplog.
oplog_truncate_after_ts = latest_oplog_entry_ts
# Check the ordering invariants before the secondary has reconciled the end of
# its oplog.
# The "oplogTruncateAfterPoint" is set to the first timestamp of each batch of
# oplog entries before they are written to the oplog. Thus, it can be ahead
# of the top of the oplog before any oplog entries are written, and behind it
# after some are written. Thus, we cannot compare it to the top of the oplog.
# appliedThrough <= minValid
# appliedThrough represents the end of the previous batch, so it is always the
# earliest.
if applied_through_ts > minvalid_ts:
raise errors.ServerFailure(
"The condition appliedThrough <= minValid ({} <= {}) doesn't hold: minValid"
" document={}, latest oplog entry={}".format(
applied_through_ts, minvalid_ts, minvalid_doc, latest_oplog_doc))
# minValid <= oplogTruncateAfterPoint
# This is true because this hook is never run after a rollback. Thus, we only
# move "minValid" to the end of each batch after the batch is written to the oplog.
# We reset the "oplogTruncateAfterPoint" to null before we move "minValid" from
# the end of the previous batch to the end of the current batch. Thus "minValid"
# must be less than or equal to the "oplogTruncateAfterPoint".
if minvalid_ts > oplog_truncate_after_ts:
raise errors.ServerFailure(
"The condition minValid <= oplogTruncateAfterPoint ({} <= {}) doesn't"
" hold: minValid document={}, oplogTruncateAfterPoint document={},"
" latest oplog entry={}".format(minvalid_ts, oplog_truncate_after_ts,
minvalid_doc, oplog_truncate_after_doc,
latest_oplog_doc))
# minvalid <= latest oplog entry
# "minValid" is set to the end of a batch after the batch is written to the oplog.
# Thus it is always less than or equal to the top of the oplog.
if minvalid_ts > latest_oplog_entry_ts:
raise errors.ServerFailure(
"The condition minValid <= top of oplog ({} <= {}) doesn't"
" hold: minValid document={}, latest oplog entry={}".format(
minvalid_ts, latest_oplog_entry_ts, minvalid_doc, latest_oplog_doc))
try:
secondary.teardown()
except errors.ServerFailure:
raise errors.ServerFailure(
"{} did not exit cleanly after being started up as a standalone".format(
secondary))
except pymongo.errors.OperationFailure as err:
self.logger.exception(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" the last stable recovery timestamp, or the latest oplog entry from the"
" mongod on port %d", secondary.port)
raise errors.ServerFailure(
"Failed to read the minValid document, the oplogTruncateAfterPoint document,"
" the last stable recovery timestamp, or the latest oplog entry from the"
" mongod on port {}: {}".format(secondary.port, err.args[0]))
finally:
# Set the secondary's options back to their original values.
secondary.mongod_options["replSet"] = replset_name
def _check_invariants_as_standalone(self, secondary):
# We remove the --replSet option in order to start the node as a standalone.
replset_name = secondary.mongod_options.pop("replSet")
try:
secondary.setup()
secondary.await_ready()
client = secondary.mongo_client()
minvalid_doc = client.local["replset.minvalid"].find_one()
latest_oplog_doc = client.local["oplog.rs"].find_one(
sort=[("$natural", pymongo.DESCENDING)])
if minvalid_doc is not None:
# Check the invariants 'begin <= minValid', 'minValid <= oplogDeletePoint', and
# 'minValid <= top of oplog' before the secondary has reconciled the end of its
# oplog.
null_ts = bson.Timestamp(0, 0)
begin_ts = minvalid_doc.get("begin", {}).get("ts", null_ts)
minvalid_ts = minvalid_doc.get("ts", begin_ts)
oplog_delete_point_ts = minvalid_doc.get(
"oplogDeleteFromPoint", minvalid_ts)
if minvalid_ts == null_ts:
# The server treats the "ts" field in the minValid document as missing when its
# value is the null timestamp.
minvalid_ts = begin_ts
if oplog_delete_point_ts == null_ts:
# The server treats the "oplogDeleteFromPoint" field as missing when its value
# is the null timestamp.
oplog_delete_point_ts = minvalid_ts
latest_oplog_entry_ts = latest_oplog_doc.get(
"ts", oplog_delete_point_ts)
if not begin_ts <= minvalid_ts:
raise errors.ServerFailure(
"The condition begin <= minValid ({} <= {}) doesn't hold: minValid"
" document={}, latest oplog entry={}".format(
begin_ts, minvalid_ts, minvalid_doc,
latest_oplog_doc))
if not minvalid_ts <= oplog_delete_point_ts:
raise errors.ServerFailure(
"The condition minValid <= oplogDeletePoint ({} <= {}) doesn't hold:"
" minValid document={}, latest oplog entry={}".format(
minvalid_ts, oplog_delete_point_ts, minvalid_doc,
latest_oplog_doc))
if not minvalid_ts <= latest_oplog_entry_ts:
raise errors.ServerFailure(
"The condition minValid <= top of oplog ({} <= {}) doesn't hold: minValid"
" document={}, latest oplog entry={}".format(
minvalid_ts, latest_oplog_entry_ts, minvalid_doc,
latest_oplog_doc))
teardown_success = secondary.teardown()
if not teardown_success:
raise errors.ServerFailure(
"{} did not exit cleanly after being started up as a standalone"
.format(secondary))
except pymongo.errors.OperationFailure as err:
self.hook_test_case.logger.exception(
"Failed to read the minValid document or the latest oplog entry from the mongod on"
" port %d", secondary.port)
raise errors.ServerFailure(
"Failed to read the minValid document or the latest oplog entry from the mongod on"
" port {}: {}".format(secondary.port, err.args[0]))
finally:
# Set the secondary's options back to their original values.
secondary.mongod_options["replSet"] = replset_name
import datetime
import bson
from common.mongo import oplog
from common import event_emitter
mo = oplog.MongoOplog(
'mongodb://*****:*****@172.16.100.150,172.16.100.151,172.16.100.152/?replicaSet=foobar',
ts=bson.Timestamp(1524735047, 1))
@event_emitter.on(mo.event_emitter, 'data')
def on_data(data):
# pass
print(data)
@event_emitter.on(mo.event_emitter, 'insert')
def on_data(data):
pass
@event_emitter.on(mo.event_emitter, 'update')
def on_data(data):
pass
@event_emitter.on(mo.event_emitter, 'delete')
def on_data(data):
pass