def test_client_options(self):
repl_set = ReplicaSet().start()
try:
config_def = {
'mainAddress': repl_set.uri,
'oplogFile': from_here('lib', 'dummy.timestamp'),
'docManagers': [
{
'docManager': 'solr_doc_manager',
'targetURL': solr_url,
'args': {
'clientOptions': {
'timeout': 100
}
}
}
]
}
config_obj = config.Config(get_config_options())
config_obj.load_json(json.dumps(config_def))
config_obj.parse_args(argv=[])
conn = connector.Connector.from_config(config_obj)
self.assertEqual(100, conn.doc_managers[0].solr.timeout)
finally:
repl_set.stop()
class TestOplogManager(unittest.TestCase):
"""Defines all the testing methods, as well as a method that sets up the
cluster
"""
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict())
def tearDown(self):
try:
self.opman.join()
except RuntimeError:
pass # OplogThread may not have been started
self.primary_conn.drop_database("test")
close_client(self.primary_conn)
self.repl_set.stop()
def test_get_oplog_cursor(self):
'''Test the get_oplog_cursor method'''
# timestamp is None - all oplog entries are returned.
cursor = self.opman.get_oplog_cursor(None)
self.assertEqual(cursor.count(),
self.primary_conn["local"]["oplog.rs"].count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.primary_conn["test"]["test"].insert_one(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.test.test.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["test"]["test"].insert_many([{
"i": i
} for i in range(2, 1002)])
oplog_cursor = self.oplog_coll.find(
{
'op': {
'$ne': 'n'
},
'ns': {
'$not': re.compile(r'\.(system|\$cmd)')
}
},
sort=[("ts", pymongo.ASCENDING)])
# initial insert + 1000 more inserts
self.assertEqual(oplog_cursor.count(), 1 + 1000)
pivot = oplog_cursor.skip(400).limit(-1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 1 + 1000 - 400)
def test_get_last_oplog_timestamp(self):
"""Test the get_last_oplog_timestamp method"""
# "empty" the oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
self.assertEqual(self.opman.get_last_oplog_timestamp(), None)
# Test non-empty oplog
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({"i": i + 500})
oplog = self.primary_conn["local"]["oplog.rs"]
oplog = oplog.find().sort("$natural", pymongo.DESCENDING).limit(-1)[0]
self.assertEqual(self.opman.get_last_oplog_timestamp(), oplog["ts"])
def test_dump_collection(self):
"""Test the dump_collection method
Cases:
1. empty oplog
2. non-empty oplog
"""
# Test with empty oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
last_ts = self.opman.dump_collection()
self.assertEqual(last_ts, None)
# Test with non-empty oplog
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({"i": i + 500})
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 1000)
def test_dump_collection_with_error(self):
"""Test the dump_collection method with invalid documents.
Cases:
1. non-empty oplog, continue_on_error=True, invalid documents
"""
# non-empty oplog, continue_on_error=True, invalid documents
self.opman.continue_on_error = True
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
docs = [{'a': i} for i in range(100)]
for i in range(50, 60):
docs[i]['_upsert_exception'] = True
self.primary_conn['test']['test'].insert_many(docs)
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
docs = self.opman.doc_managers[0]._search()
docs.sort(key=lambda doc: doc['a'])
self.assertEqual(len(docs), 90)
expected_a = itertools.chain(range(0, 50), range(60, 100))
for doc, correct_a in zip(docs, expected_a):
self.assertEqual(doc['a'], correct_a)
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
self.opman.collection_dump = False
self.assertTrue(
all(doc['op'] == 'n' for doc in self.opman.init_cursor()[0]))
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman.collection_dump = True
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor, None)
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman.oplog = self.primary_conn['local']['oplog.rs']
collection = self.primary_conn["test"]["test"]
collection.insert_one({"i": 1})
collection.delete_one({"i": 1})
time.sleep(3)
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, last_ts)
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts)
# No last checkpoint, no collection dump, something in oplog
self.opman.oplog_progress = LockingDict()
self.opman.collection_dump = False
collection.insert_one({"i": 2})
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
for i in range(cursor_len - 1):
next(cursor)
self.assertEqual(next(cursor)['o']['i'], 2)
self.assertEqual(self.opman.checkpoint, last_ts)
# Last checkpoint exists
progress = LockingDict()
self.opman.oplog_progress = progress
for i in range(1000):
collection.insert_one({"i": i + 500})
entry = list(self.primary_conn["local"]["oplog.rs"].find(skip=200,
limit=-2))
progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"]
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(next(cursor)["ts"], entry[1]["ts"])
self.assertEqual(self.opman.checkpoint, entry[0]["ts"])
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)],
entry[0]["ts"])
# Last checkpoint is behind
progress = LockingDict()
progress.get_dict()[str(self.opman.oplog)] = bson.Timestamp(1, 0)
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman.checkpoint)
def test_namespace_mapping(self):
"""Test mapping of namespaces
Cases:
upsert/delete/update of documents:
1. in namespace set, mapping provided
2. outside of namespace set, mapping provided
"""
source_ns = ["test.test1", "test.test2"]
phony_ns = ["test.phony1", "test.phony2"]
dest_mapping = {
"test.test1": "test.test1_dest",
"test.test2": "test.test2_dest"
}
self.opman.dest_mapping = dest_mapping
self.opman.namespace_set = source_ns
docman = self.opman.doc_managers[0]
# start replicating
self.opman.start()
base_doc = {"_id": 1, "name": "superman"}
# doc in namespace set
for ns in source_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
assert_soon(lambda: len(docman._search()) == 1)
self.assertEqual(docman._search()[0]["ns"], dest_mapping[ns])
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1}, {"$set": {
"weakness": "kryptonite"
}})
def update_complete():
docs = docman._search()
for d in docs:
if d.get("weakness") == "kryptonite":
return True
return False
assert_soon(update_complete)
self.assertEqual(docman._search()[0]["ns"], dest_mapping[ns])
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test delete
self.primary_conn[db][coll].delete_one({"_id": 1})
assert_soon(lambda: len(docman._search()) == 0)
bad = [d for d in docman._search() if d["ns"] == dest_mapping[ns]]
self.assertEqual(len(bad), 0)
# cleanup
self.primary_conn[db][coll].delete_many({})
self.opman.doc_managers[0]._delete()
# doc not in namespace set
for ns in phony_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1}, {"$set": {
"weakness": "kryptonite"
}})
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert_one({
"name": "kermit",
"color": "green"
})
self.primary_conn["test"]["test"].insert_one({
"name": "elmo",
"color": "firetruck red"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets")
self.primary_conn["test"]["test"].delete_one({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets")
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
class TestOplogManager(unittest.TestCase):
"""Defines all the testing methods, as well as a method that sets up the
cluster
"""
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict()
)
def tearDown(self):
try:
self.opman.join()
except RuntimeError:
pass # OplogThread may not have been started
self.primary_conn.drop_database("test")
close_client(self.primary_conn)
self.repl_set.stop()
def test_get_oplog_cursor(self):
'''Test the get_oplog_cursor method'''
# timestamp is None - all oplog entries are returned.
cursor = self.opman.get_oplog_cursor(None)
self.assertEqual(cursor.count(),
self.primary_conn["local"]["oplog.rs"].count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "i": 1}
self.primary_conn["test"]["test"].insert_one(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.test.test.find_one(next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["test"]["test"].insert_many(
[{"i": i} for i in range(2, 1002)])
oplog_cursor = self.oplog_coll.find(
{'op': {'$ne': 'n'},
'ns': {'$not': re.compile(r'\.(system|\$cmd)')}},
sort=[("ts", pymongo.ASCENDING)]
)
# initial insert + 1000 more inserts
self.assertEqual(oplog_cursor.count(), 1 + 1000)
pivot = oplog_cursor.skip(400).limit(-1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 1 + 1000 - 400)
def test_get_last_oplog_timestamp(self):
"""Test the get_last_oplog_timestamp method"""
# "empty" the oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
self.assertEqual(self.opman.get_last_oplog_timestamp(), None)
# Test non-empty oplog
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({
"i": i + 500
})
oplog = self.primary_conn["local"]["oplog.rs"]
oplog = oplog.find().sort("$natural", pymongo.DESCENDING).limit(-1)[0]
self.assertEqual(self.opman.get_last_oplog_timestamp(),
oplog["ts"])
def test_dump_collection(self):
"""Test the dump_collection method
Cases:
1. empty oplog
2. non-empty oplog
"""
# Test with empty oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
last_ts = self.opman.dump_collection()
self.assertEqual(last_ts, None)
# Test with non-empty oplog
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({
"i": i + 500
})
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 1000)
def test_dump_collection_with_error(self):
"""Test the dump_collection method with invalid documents.
Cases:
1. non-empty oplog, continue_on_error=True, invalid documents
"""
# non-empty oplog, continue_on_error=True, invalid documents
self.opman.continue_on_error = True
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
docs = [{'a': i} for i in range(100)]
for i in range(50, 60):
docs[i]['_upsert_exception'] = True
self.primary_conn['test']['test'].insert_many(docs)
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
docs = self.opman.doc_managers[0]._search()
docs.sort(key=lambda doc: doc['a'])
self.assertEqual(len(docs), 90)
expected_a = itertools.chain(range(0, 50), range(60, 100))
for doc, correct_a in zip(docs, expected_a):
self.assertEqual(doc['a'], correct_a)
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
self.opman.collection_dump = False
self.assertTrue(all(doc['op'] == 'n'
for doc in self.opman.init_cursor()[0]))
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman.collection_dump = True
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor, None)
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman.oplog = self.primary_conn['local']['oplog.rs']
collection = self.primary_conn["test"]["test"]
collection.insert_one({"i": 1})
collection.delete_one({"i": 1})
time.sleep(3)
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(self.opman.checkpoint, last_ts)
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)], last_ts)
# No last checkpoint, no collection dump, something in oplog
self.opman.oplog_progress = LockingDict()
self.opman.collection_dump = False
collection.insert_one({"i": 2})
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_len = self.opman.init_cursor()
for i in range(cursor_len - 1):
next(cursor)
self.assertEqual(next(cursor)['o']['i'], 2)
self.assertEqual(self.opman.checkpoint, last_ts)
# Last checkpoint exists
progress = LockingDict()
self.opman.oplog_progress = progress
for i in range(1000):
collection.insert_one({"i": i + 500})
entry = list(
self.primary_conn["local"]["oplog.rs"].find(skip=200, limit=-2))
progress.get_dict()[str(self.opman.oplog)] = entry[0]["ts"]
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(next(cursor)["ts"], entry[1]["ts"])
self.assertEqual(self.opman.checkpoint, entry[0]["ts"])
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[str(self.opman.oplog)],
entry[0]["ts"])
# Last checkpoint is behind
progress = LockingDict()
progress.get_dict()[str(self.opman.oplog)] = bson.Timestamp(1, 0)
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_len = self.opman.init_cursor()
self.assertEqual(cursor_len, 0)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman.checkpoint)
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.primary_client
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {
"a": 1, "b": 2, "c": 3,
"d": 4, "e": 5, "f": 6,
"_id": 1
}
db = conn['test']['test']
db.insert_one(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_namespace_mapping(self):
"""Test mapping of namespaces
Cases:
upsert/delete/update of documents:
1. in namespace set, mapping provided
2. outside of namespace set, mapping provided
"""
source_ns = ["test.test1", "test.test2"]
phony_ns = ["test.phony1", "test.phony2"]
dest_mapping = {"test.test1": "test.test1_dest",
"test.test2": "test.test2_dest"}
self.opman.dest_mapping = dest_mapping
self.opman.namespace_set = source_ns
docman = self.opman.doc_managers[0]
# start replicating
self.opman.start()
base_doc = {"_id": 1, "name": "superman"}
# doc in namespace set
for ns in source_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
assert_soon(lambda: len(docman._search()) == 1)
self.assertEqual(docman._search()[0]["ns"], dest_mapping[ns])
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1},
{"$set": {"weakness": "kryptonite"}}
)
def update_complete():
docs = docman._search()
for d in docs:
if d.get("weakness") == "kryptonite":
return True
return False
assert_soon(update_complete)
self.assertEqual(docman._search()[0]["ns"], dest_mapping[ns])
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test delete
self.primary_conn[db][coll].delete_one({"_id": 1})
assert_soon(lambda: len(docman._search()) == 0)
bad = [d for d in docman._search()
if d["ns"] == dest_mapping[ns]]
self.assertEqual(len(bad), 0)
# cleanup
self.primary_conn[db][coll].delete_many({})
self.opman.doc_managers[0]._delete()
# doc not in namespace set
for ns in phony_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1},
{"$set": {"weakness": "kryptonite"}}
)
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert_one({
"name": "kermit",
"color": "green"
})
self.primary_conn["test"]["test"].insert_one({
"name": "elmo",
"color": "firetruck red"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets"
)
self.primary_conn["test"]["test"].delete_one({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets"
)
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_filter_oplog_entry(self):
# Test oplog entries: these are callables, since
# filter_oplog_entry modifies the oplog entry in-place
insert_op = lambda: {
"op": "i",
"o": {
"_id": 0,
"a": 1,
"b": 2,
"c": 3
}
}
update_op = lambda: {
"op": "u",
"o": {
"$set": {
"a": 4,
"b": 5
},
"$unset": {
"c": True
}
},
"o2": {
"_id": 1
}
}
# Case 0: insert op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered, insert_op())
# Case 1: insert op, fields provided
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0, 'a': 1, 'b': 2})
# Case 2: insert op, fields provided, doc becomes empty except for _id
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0})
# Case 3: update op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, update_op())
# Case 4: update op, fields provided
self.opman.fields = ['a', 'c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('b', filtered['o']['$set'])
self.assertIn('a', filtered['o']['$set'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
# Case 5: update op, fields provided, empty $set
self.opman.fields = ['c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$set', filtered['o'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
# Case 6: update op, fields provided, empty $unset
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$unset', filtered['o'])
self.assertEqual(filtered['o']['$set'], update_op()['o']['$set'])
# Case 7: update op, fields provided, entry is nullified
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, None)
# Case 8: update op, fields provided, replacement
self.opman.fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry({
'op': 'u',
'o': {'a': 1, 'b': 2, 'c': 3, 'd': 4}
})
self.assertEqual(
filtered, {'op': 'u', 'o': {'a': 1, 'b': 2, 'c': 3}})
def test_fields(self):
fields = ["_id","title","content","author"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields = fields
)
self.assertEqual(set(fields), opman._fields)
self.assertEqual(sorted(fields), sorted(opman.fields))
class TestFilterFields(unittest.TestCase):
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict()
)
def tearDown(self):
try:
self.opman.join()
except RuntimeError:
pass # OplogThread may not have been started
self.primary_conn.drop_database("test")
close_client(self.primary_conn)
self.repl_set.stop()
def _check_fields(self, opman, fields, exclude_fields, projection):
if fields:
self.assertEqual(sorted(opman.fields), sorted(fields))
self.assertEqual(opman._fields, set(fields))
else:
self.assertEqual(opman.fields, None)
self.assertEqual(opman._fields, set([]))
if exclude_fields:
self.assertEqual(sorted(opman.exclude_fields), sorted(exclude_fields))
self.assertEqual(opman._exclude_fields, set(exclude_fields))
else:
self.assertEqual(opman.exclude_fields, None)
self.assertEqual(opman._exclude_fields, set([]))
self.assertEqual(opman._projection, projection)
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.primary_client
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {
"a": 1, "b": 2, "c": 3,
"d": 4, "e": 5, "f": 6,
"_id": 1
}
db = conn['test']['test']
db.insert_one(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_filter_exclude_oplog_entry(self):
# Test oplog entries: these are callables, since
# filter_oplog_entry modifies the oplog entry in-place
insert_op = lambda: {
"op": "i",
"o": {
"_id": 0,
"a": 1,
"b": 2,
"c": 3
}
}
update_op = lambda: {
"op": "u",
"o": {
"$set": {
"a": 4,
"b": 5
},
"$unset": {
"c": True
}
},
"o2": {
"_id": 1
}
}
# Case 0: insert op, no fields provided
self.opman.exclude_fields = None
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered, insert_op())
self.assertEqual(None, self.opman._projection)
# Case 1: insert op, fields provided
self.opman.exclude_fields = ['c']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0, 'a': 1, 'b': 2})
self.assertEqual({'c': 0}, self.opman._projection)
# Case 2: insert op, fields provided, doc becomes empty except for _id
self.opman.exclude_fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0})
self.assertEqual({'a': 0, 'b': 0, 'c': 0}, self.opman._projection)
# Case 3: update op, no fields provided
self.opman.exclude_fields = None
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, update_op())
self.assertEqual(None, self.opman._projection)
# Case 4: update op, fields provided
self.opman.exclude_fields = ['b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('b', filtered['o']['$set'])
self.assertIn('a', filtered['o']['$set'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'b': 0}, self.opman._projection)
# Case 5: update op, fields provided, empty $set
self.opman.exclude_fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$set', filtered['o'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'a': 0, 'b': 0}, self.opman._projection)
# Case 6: update op, fields provided, empty $unset
self.opman.exclude_fields = ['c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$unset', filtered['o'])
self.assertEqual(filtered['o']['$set'], update_op()['o']['$set'])
self.assertEqual({'c': 0}, self.opman._projection)
# Case 7: update op, fields provided, entry is nullified
self.opman.exclude_fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, None)
self.assertEqual({'a': 0, 'b': 0, 'c': 0}, self.opman._projection)
# Case 8: update op, fields provided, replacement
self.opman.exclude_fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry({
'op': 'u',
'o': {'a': 1, 'b': 2, 'c': 3, 'd': 4}
})
self.assertEqual(
filtered, {'op': 'u', 'o': {'a': 1, 'b': 2, 'c': 3}})
self.assertEqual({'d': 0, 'e': 0, 'f': 0}, self.opman._projection)
def test_filter_oplog_entry(self):
# Test oplog entries: these are callables, since
# filter_oplog_entry modifies the oplog entry in-place
insert_op = lambda: {
"op": "i",
"o": {
"_id": 0,
"a": 1,
"b": 2,
"c": 3
}
}
update_op = lambda: {
"op": "u",
"o": {
"$set": {
"a": 4,
"b": 5
},
"$unset": {
"c": True
}
},
"o2": {
"_id": 1
}
}
# Case 0: insert op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered, insert_op())
self.assertEqual(None, self.opman._projection)
# Case 1: insert op, fields provided
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0, 'a': 1, 'b': 2})
self.assertEqual({'_id': 1, 'a': 1, 'b': 1}, self.opman._projection)
# Case 2: insert op, fields provided, doc becomes empty except for _id
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0})
self.assertEqual({'_id': 1, 'd': 1, 'e': 1, 'f': 1},
self.opman._projection)
# Case 3: update op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, update_op())
self.assertEqual(None, self.opman._projection)
# Case 4: update op, fields provided
self.opman.fields = ['a', 'c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('b', filtered['o']['$set'])
self.assertIn('a', filtered['o']['$set'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'_id': 1, 'a': 1, 'c': 1}, self.opman._projection)
# Case 5: update op, fields provided, empty $set
self.opman.fields = ['c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$set', filtered['o'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'_id': 1, 'c': 1}, self.opman._projection)
# Case 6: update op, fields provided, empty $unset
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$unset', filtered['o'])
self.assertEqual(filtered['o']['$set'], update_op()['o']['$set'])
self.assertEqual({'_id': 1, 'a': 1, 'b': 1}, self.opman._projection)
# Case 7: update op, fields provided, entry is nullified
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, None)
self.assertEqual({'_id': 1, 'd': 1, 'e': 1, 'f': 1},
self.opman._projection)
# Case 8: update op, fields provided, replacement
self.opman.fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry({
'op': 'u',
'o': {'a': 1, 'b': 2, 'c': 3, 'd': 4}
})
self.assertEqual(
filtered, {'op': 'u', 'o': {'a': 1, 'b': 2, 'c': 3}})
self.assertEqual({'_id': 1, 'a': 1, 'b': 1, 'c': 1},
self.opman._projection)
def test_exclude_fields_constructor(self):
# Test with the "_id" field in exclude_fields
exclude_fields = ["_id", "title", "content", "author"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields
)
exclude_fields.remove('_id')
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered)
# Test without "_id" field included in exclude_fields
exclude_fields = ["title", "content", "author"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields
)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = extra_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual({'extra1': 1, 'extra2': 1}, filtered)
# Test with only "_id" field in exclude_fields
exclude_fields = ["_id"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields
)
self._check_fields(opman, [], [], None)
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
# Test with nothing set for exclude_fields
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
exclude_fields=None
)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_constructor(self):
# Test with "_id" field in constructor
fields = ["_id", "title", "content", "author"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields=fields
)
self._check_fields(opman, fields, [],
dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test without "_id" field in constructor
fields = ["title", "content", "author"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields = fields
)
fields.append('_id')
self._check_fields(opman, fields, [],
dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test with only "_id" field
fields = ["_id"]
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields = fields
)
self._check_fields(opman, fields, [],
dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual({'_id': 1}, filtered)
# Test with no fields set
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_exclude_fields_attr(self):
# Test with the "_id" field in exclude_fields.
exclude_fields = ["_id", "title", "content", "author"]
exclude_fields.remove('_id')
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered)
# Test without "_id" field included in exclude_fields
exclude_fields = ["title", "content", "author"]
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = extra_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual({'extra1': 1, 'extra2': 1}, filtered)
# Test with only "_id" field in exclude_fields
exclude_fields = ["_id"]
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], [], None)
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
# Test with nothing set for exclude_fields
self.opman.exclude_fields = None
self._check_fields(self.opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_attr(self):
# Test with "_id" field included in fields
fields = ["_id", "title", "content", "author"]
self.opman.fields = fields
self._check_fields(self.opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test without "_id" field included in fields
fields = ["title", "content", "author"]
self.opman.fields = fields
fields.append('_id')
self._check_fields(self.opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test with only "_id" field
fields = ["_id"]
self.opman.fields = fields
self._check_fields(self.opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual({'_id': 1}, filtered)
# Test with no fields set
self.opman.fields = None
self._check_fields(self.opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = self.opman.filter_oplog_entry(
{'op': 'i',
'o': dict((f, 1) for f in extra_fields)})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_nested_fields(self):
def check_nested(document, fields, filtered_document):
self.opman.fields = fields
fields.append('_id')
self.assertEqual(set(fields), self.opman._fields)
self.assertEqual(sorted(fields), sorted(self.opman.fields))
filtered_result = self.opman.filter_oplog_entry(
{'op': 'i',
'o': document})['o']
self.assertEqual(filtered_result, filtered_document)
document = {'name': 'Han Solo', 'a': {'b': {}}}
fields = ['name', 'a.b.c']
filtered_document = {'name': 'Han Solo'}
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': 2, 'e': 3}, 'e': 5},
'b': 2,
'c': {'g': 1}}
fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'c': 2}, 'e': 5}}
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': 2, 'e': 3}, 'e': 5},
'b': 2,
'c': {'g': 1},
'_id': 1}
fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'c': 2}, 'e': 5}, '_id': 1}
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
fields = ['a.b', '-a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
fields = ['a', '-a.-b']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}},
'_id': 1}
fields = ['a.b', '-a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
fields = ['a', '-a.-b']
check_nested(document, fields, filtered_document)
document = {'test': 1}
fields = ['doesnt_exist']
filtered_document = {}
check_nested(document, fields, filtered_document)
document = {'a': {'b': 1}, 'b': {'a': 1}}
fields = ['a.b', 'b.a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'a': {'b': 1}}}, 'c': {'a': {'b': 1}}}
fields = ['a.b']
filtered_document = {'a': {'b': {'a': {'b': 1}}}}
check_nested(document, fields, filtered_document)
document = {'name': 'anna', 'name_of_cat': 'pushkin'}
fields = ['name']
filtered_document = {'name': 'anna'}
check_nested(document, fields, filtered_document)
def test_nested_exclude_fields(self):
def check_nested(document, exclude_fields, filtered_document):
self.opman.exclude_fields = exclude_fields
if '_id' in exclude_fields:
exclude_fields.remove('_id')
self.assertEqual(set(exclude_fields), self.opman._exclude_fields)
self.assertEqual(sorted(exclude_fields),
sorted(self.opman.exclude_fields))
filtered_result = self.opman.filter_oplog_entry(
{'op': 'i',
'o': document})['o']
self.assertEqual(filtered_result, filtered_document)
document = {'a': {'b': {'c': {'d': 0, 'e': 1}}}}
exclude_fields = ['a.b.c.d']
filtered_document = {'a': {'b': {'c': {'e': 1}}}}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'-a': 0, 'd': {'e': {'f': 1}}}}}}
exclude_fields = ['a.b.c.d.e.f']
filtered_document = {'a': {'b': {'c': {'-a': 0}}}}
check_nested(document, exclude_fields, filtered_document)
document = {'a': 1}
exclude_fields = ['a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': 2, 'e': 3}, 'e': 5},
'b': 2,
'c': {'g': 1}}
exclude_fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'e': 3}},
'b': 2,
'c': {'g': 1}}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': 2, 'e': 3}, 'e': 5},
'b': 2,
'c': {'g': 1},
'_id': 1}
exclude_fields = ['a.b.c', 'a.e', '_id']
filtered_document = {'a': {'b': {'e': 3}},
'b': 2, 'c': {'g': 1},
'_id': 1}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}},
'-a': {'-b': {'-c': 2}}}
exclude_fields = ['a.b', '-a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}},
'-a': {'-b': {'-c': 2}}}
exclude_fields = ['a', '-a.-b']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}},
'-a': {'-b': {'-c': 2}},
'_id': 1}
exclude_fields = ['a.b', '-a']
filtered_document = {'_id': 1}
check_nested(document, exclude_fields, filtered_document)
document = {'test': 1}
exclude_fields = ['doesnt_exist']
filtered_document = document.copy()
check_nested(document, exclude_fields, filtered_document)
document = {'test': 1}
exclude_fields = ['test.doesnt_exist']
filtered_document = document.copy()
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': 1}, 'b': {'a': 1}}
exclude_fields = ['a.b', 'b.a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'a': {'b': 1}}}, 'c': {'a': {'b': 1}}}
exclude_fields = ['a.b']
filtered_document = {'c': {'a': {'b': 1}}}
check_nested(document, exclude_fields, filtered_document)
document = {'name': 'anna', 'name_of_cat': 'pushkin'}
exclude_fields = ['name']
filtered_document = {'name_of_cat': 'pushkin'}
check_nested(document, exclude_fields, filtered_document)
def test_fields_and_exclude(self):
fields = ['a', 'b', 'c', '_id']
exclude_fields = ['x', 'y', 'z']
# Test setting both to None in constructor
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields=None,
exclude_fields=None
)
self._check_fields(opman, [], [], None)
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
fields=None,
exclude_fields=exclude_fields
)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
# Test setting fields when exclude_fields is set
self.assertRaises(
errors.InvalidConfiguration, setattr, opman, "fields", fields)
self.assertRaises(
errors.InvalidConfiguration, setattr, opman, "fields", None)
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
exclude_fields=None,
fields=fields
)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", exclude_fields)
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", None)
self.assertRaises(
errors.InvalidConfiguration, OplogThread,
self.primary_conn,
(DocManager(),),
LockingDict(),
fields=fields,
exclude_fields=exclude_fields)
class TestFilterFields(unittest.TestCase):
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict())
def tearDown(self):
try:
self.opman.join()
except RuntimeError:
pass # OplogThread may not have been started
self.primary_conn.drop_database("test")
close_client(self.primary_conn)
self.repl_set.stop()
def _check_fields(self, opman, fields, exclude_fields, projection):
if fields:
self.assertEqual(sorted(opman.fields), sorted(fields))
self.assertEqual(opman._fields, set(fields))
else:
self.assertEqual(opman.fields, None)
self.assertEqual(opman._fields, set([]))
if exclude_fields:
self.assertEqual(sorted(opman.exclude_fields),
sorted(exclude_fields))
self.assertEqual(opman._exclude_fields, set(exclude_fields))
else:
self.assertEqual(opman.exclude_fields, None)
self.assertEqual(opman._exclude_fields, set([]))
self.assertEqual(opman._projection, projection)
def test_filter_fields(self):
docman = self.opman.doc_managers[0]
conn = self.opman.primary_client
include_fields = ["a", "b", "c"]
exclude_fields = ["d", "e", "f"]
# Set fields to care about
self.opman.fields = include_fields
# Documents have more than just these fields
doc = {"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "_id": 1}
db = conn['test']['test']
db.insert_one(doc)
assert_soon(lambda: db.count() == 1)
self.opman.dump_collection()
result = docman._search()[0]
keys = result.keys()
for inc, exc in zip(include_fields, exclude_fields):
self.assertIn(inc, keys)
self.assertNotIn(exc, keys)
def test_filter_exclude_oplog_entry(self):
# Test oplog entries: these are callables, since
# filter_oplog_entry modifies the oplog entry in-place
insert_op = lambda: {
"op": "i",
"o": {
"_id": 0,
"a": 1,
"b": 2,
"c": 3
}
}
update_op = lambda: {
"op": "u",
"o": {
"$set": {
"a": 4,
"b": 5
},
"$unset": {
"c": True
}
},
"o2": {
"_id": 1
}
}
# Case 0: insert op, no fields provided
self.opman.exclude_fields = None
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered, insert_op())
self.assertEqual(None, self.opman._projection)
# Case 1: insert op, fields provided
self.opman.exclude_fields = ['c']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0, 'a': 1, 'b': 2})
self.assertEqual({'c': 0}, self.opman._projection)
# Case 2: insert op, fields provided, doc becomes empty except for _id
self.opman.exclude_fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0})
self.assertEqual({'a': 0, 'b': 0, 'c': 0}, self.opman._projection)
# Case 3: update op, no fields provided
self.opman.exclude_fields = None
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, update_op())
self.assertEqual(None, self.opman._projection)
# Case 4: update op, fields provided
self.opman.exclude_fields = ['b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('b', filtered['o']['$set'])
self.assertIn('a', filtered['o']['$set'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'b': 0}, self.opman._projection)
# Case 5: update op, fields provided, empty $set
self.opman.exclude_fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$set', filtered['o'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'a': 0, 'b': 0}, self.opman._projection)
# Case 6: update op, fields provided, empty $unset
self.opman.exclude_fields = ['c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$unset', filtered['o'])
self.assertEqual(filtered['o']['$set'], update_op()['o']['$set'])
self.assertEqual({'c': 0}, self.opman._projection)
# Case 7: update op, fields provided, entry is nullified
self.opman.exclude_fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, None)
self.assertEqual({'a': 0, 'b': 0, 'c': 0}, self.opman._projection)
# Case 8: update op, fields provided, replacement
self.opman.exclude_fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry({
'op': 'u',
'o': {
'a': 1,
'b': 2,
'c': 3,
'd': 4
}
})
self.assertEqual(filtered, {'op': 'u', 'o': {'a': 1, 'b': 2, 'c': 3}})
self.assertEqual({'d': 0, 'e': 0, 'f': 0}, self.opman._projection)
def test_filter_oplog_entry(self):
# Test oplog entries: these are callables, since
# filter_oplog_entry modifies the oplog entry in-place
insert_op = lambda: {
"op": "i",
"o": {
"_id": 0,
"a": 1,
"b": 2,
"c": 3
}
}
update_op = lambda: {
"op": "u",
"o": {
"$set": {
"a": 4,
"b": 5
},
"$unset": {
"c": True
}
},
"o2": {
"_id": 1
}
}
# Case 0: insert op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered, insert_op())
self.assertEqual(None, self.opman._projection)
# Case 1: insert op, fields provided
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0, 'a': 1, 'b': 2})
self.assertEqual({'_id': 1, 'a': 1, 'b': 1}, self.opman._projection)
# Case 2: insert op, fields provided, doc becomes empty except for _id
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(insert_op())
self.assertEqual(filtered['o'], {'_id': 0})
self.assertEqual({
'_id': 1,
'd': 1,
'e': 1,
'f': 1
}, self.opman._projection)
# Case 3: update op, no fields provided
self.opman.fields = None
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, update_op())
self.assertEqual(None, self.opman._projection)
# Case 4: update op, fields provided
self.opman.fields = ['a', 'c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('b', filtered['o']['$set'])
self.assertIn('a', filtered['o']['$set'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'_id': 1, 'a': 1, 'c': 1}, self.opman._projection)
# Case 5: update op, fields provided, empty $set
self.opman.fields = ['c']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$set', filtered['o'])
self.assertEqual(filtered['o']['$unset'], update_op()['o']['$unset'])
self.assertEqual({'_id': 1, 'c': 1}, self.opman._projection)
# Case 6: update op, fields provided, empty $unset
self.opman.fields = ['a', 'b']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertNotIn('$unset', filtered['o'])
self.assertEqual(filtered['o']['$set'], update_op()['o']['$set'])
self.assertEqual({'_id': 1, 'a': 1, 'b': 1}, self.opman._projection)
# Case 7: update op, fields provided, entry is nullified
self.opman.fields = ['d', 'e', 'f']
filtered = self.opman.filter_oplog_entry(update_op())
self.assertEqual(filtered, None)
self.assertEqual({
'_id': 1,
'd': 1,
'e': 1,
'f': 1
}, self.opman._projection)
# Case 8: update op, fields provided, replacement
self.opman.fields = ['a', 'b', 'c']
filtered = self.opman.filter_oplog_entry({
'op': 'u',
'o': {
'a': 1,
'b': 2,
'c': 3,
'd': 4
}
})
self.assertEqual(filtered, {'op': 'u', 'o': {'a': 1, 'b': 2, 'c': 3}})
self.assertEqual({
'_id': 1,
'a': 1,
'b': 1,
'c': 1
}, self.opman._projection)
def test_exclude_fields_constructor(self):
# Test with the "_id" field in exclude_fields
exclude_fields = ["_id", "title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields)
exclude_fields.remove('_id')
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered)
# Test without "_id" field included in exclude_fields
exclude_fields = ["title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = extra_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'extra1': 1, 'extra2': 1}, filtered)
# Test with only "_id" field in exclude_fields
exclude_fields = ["_id"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
exclude_fields=exclude_fields)
self._check_fields(opman, [], [], None)
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
# Test with nothing set for exclude_fields
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
exclude_fields=None)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_constructor(self):
# Test with "_id" field in constructor
fields = ["_id", "title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test without "_id" field in constructor
fields = ["title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
fields=fields)
fields.append('_id')
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test with only "_id" field
fields = ["_id"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'_id': 1}, filtered)
# Test with no fields set
opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_exclude_fields_attr(self):
# Test with the "_id" field in exclude_fields.
exclude_fields = ["_id", "title", "content", "author"]
exclude_fields.remove('_id')
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered)
# Test without "_id" field included in exclude_fields
exclude_fields = ["title", "content", "author"]
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = extra_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'extra1': 1, 'extra2': 1}, filtered)
# Test with only "_id" field in exclude_fields
exclude_fields = ["_id"]
self.opman.exclude_fields = exclude_fields
self._check_fields(self.opman, [], [], None)
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
# Test with nothing set for exclude_fields
self.opman.exclude_fields = None
self._check_fields(self.opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_attr(self):
# Test with "_id" field included in fields
fields = ["_id", "title", "content", "author"]
self.opman.fields = fields
self._check_fields(self.opman, fields, [], dict(
(f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test without "_id" field included in fields
fields = ["title", "content", "author"]
self.opman.fields = fields
fields.append('_id')
self._check_fields(self.opman, fields, [], dict(
(f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test with only "_id" field
fields = ["_id"]
self.opman.fields = fields
self._check_fields(self.opman, fields, [], dict(
(f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'_id': 1}, filtered)
# Test with no fields set
self.opman.fields = None
self._check_fields(self.opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = self.opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_nested_fields(self):
def check_nested(document, fields, filtered_document):
self.opman.fields = fields
fields.append('_id')
self.assertEqual(set(fields), self.opman._fields)
self.assertEqual(sorted(fields), sorted(self.opman.fields))
filtered_result = self.opman.filter_oplog_entry({
'op': 'i',
'o': document
})['o']
self.assertEqual(filtered_result, filtered_document)
document = {'name': 'Han Solo', 'a': {'b': {}}}
fields = ['name', 'a.b.c']
filtered_document = {'name': 'Han Solo'}
check_nested(document, fields, filtered_document)
document = {
'a': {
'b': {
'c': 2,
'e': 3
},
'e': 5
},
'b': 2,
'c': {
'g': 1
}
}
fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'c': 2}, 'e': 5}}
check_nested(document, fields, filtered_document)
document = {
'a': {
'b': {
'c': 2,
'e': 3
},
'e': 5
},
'b': 2,
'c': {
'g': 1
},
'_id': 1
}
fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'c': 2}, 'e': 5}, '_id': 1}
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
fields = ['a.b', '-a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
fields = ['a', '-a.-b']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {
'a': {
'b': {
'c': {
'd': 1
}
}
},
'-a': {
'-b': {
'-c': 2
}
},
'_id': 1
}
fields = ['a.b', '-a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
fields = ['a', '-a.-b']
check_nested(document, fields, filtered_document)
document = {'test': 1}
fields = ['doesnt_exist']
filtered_document = {}
check_nested(document, fields, filtered_document)
document = {'a': {'b': 1}, 'b': {'a': 1}}
fields = ['a.b', 'b.a']
filtered_document = document.copy()
check_nested(document, fields, filtered_document)
document = {'a': {'b': {'a': {'b': 1}}}, 'c': {'a': {'b': 1}}}
fields = ['a.b']
filtered_document = {'a': {'b': {'a': {'b': 1}}}}
check_nested(document, fields, filtered_document)
document = {'name': 'anna', 'name_of_cat': 'pushkin'}
fields = ['name']
filtered_document = {'name': 'anna'}
check_nested(document, fields, filtered_document)
def test_nested_exclude_fields(self):
def check_nested(document, exclude_fields, filtered_document):
self.opman.exclude_fields = exclude_fields
if '_id' in exclude_fields:
exclude_fields.remove('_id')
self.assertEqual(set(exclude_fields), self.opman._exclude_fields)
self.assertEqual(sorted(exclude_fields),
sorted(self.opman.exclude_fields))
filtered_result = self.opman.filter_oplog_entry({
'op': 'i',
'o': document
})['o']
self.assertEqual(filtered_result, filtered_document)
document = {'a': {'b': {'c': {'d': 0, 'e': 1}}}}
exclude_fields = ['a.b.c.d']
filtered_document = {'a': {'b': {'c': {'e': 1}}}}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'-a': 0, 'd': {'e': {'f': 1}}}}}}
exclude_fields = ['a.b.c.d.e.f']
filtered_document = {'a': {'b': {'c': {'-a': 0}}}}
check_nested(document, exclude_fields, filtered_document)
document = {'a': 1}
exclude_fields = ['a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {
'a': {
'b': {
'c': 2,
'e': 3
},
'e': 5
},
'b': 2,
'c': {
'g': 1
}
}
exclude_fields = ['a.b.c', 'a.e']
filtered_document = {'a': {'b': {'e': 3}}, 'b': 2, 'c': {'g': 1}}
check_nested(document, exclude_fields, filtered_document)
document = {
'a': {
'b': {
'c': 2,
'e': 3
},
'e': 5
},
'b': 2,
'c': {
'g': 1
},
'_id': 1
}
exclude_fields = ['a.b.c', 'a.e', '_id']
filtered_document = {
'a': {
'b': {
'e': 3
}
},
'b': 2,
'c': {
'g': 1
},
'_id': 1
}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
exclude_fields = ['a.b', '-a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'c': {'d': 1}}}, '-a': {'-b': {'-c': 2}}}
exclude_fields = ['a', '-a.-b']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {
'a': {
'b': {
'c': {
'd': 1
}
}
},
'-a': {
'-b': {
'-c': 2
}
},
'_id': 1
}
exclude_fields = ['a.b', '-a']
filtered_document = {'_id': 1}
check_nested(document, exclude_fields, filtered_document)
document = {'test': 1}
exclude_fields = ['doesnt_exist']
filtered_document = document.copy()
check_nested(document, exclude_fields, filtered_document)
document = {'test': 1}
exclude_fields = ['test.doesnt_exist']
filtered_document = document.copy()
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': 1}, 'b': {'a': 1}}
exclude_fields = ['a.b', 'b.a']
filtered_document = {}
check_nested(document, exclude_fields, filtered_document)
document = {'a': {'b': {'a': {'b': 1}}}, 'c': {'a': {'b': 1}}}
exclude_fields = ['a.b']
filtered_document = {'c': {'a': {'b': 1}}}
check_nested(document, exclude_fields, filtered_document)
document = {'name': 'anna', 'name_of_cat': 'pushkin'}
exclude_fields = ['name']
filtered_document = {'name_of_cat': 'pushkin'}
check_nested(document, exclude_fields, filtered_document)
def test_fields_and_exclude(self):
fields = ['a', 'b', 'c', '_id']
exclude_fields = ['x', 'y', 'z']
# Test setting both to None in constructor
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
fields=None,
exclude_fields=None)
self._check_fields(opman, [], [], None)
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
fields=None,
exclude_fields=exclude_fields)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
# Test setting fields when exclude_fields is set
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"fields", fields)
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"fields", None)
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
exclude_fields=None,
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", exclude_fields)
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", None)
self.assertRaises(errors.InvalidConfiguration,
OplogThread,
self.primary_conn, (DocManager(), ),
LockingDict(),
fields=fields,
exclude_fields=exclude_fields)
class TestCommandReplication(unittest.TestCase):
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_progress = LockingDict()
self.opman = None
def tearDown(self):
try:
if self.opman:
self.opman.join()
except RuntimeError:
pass
close_client(self.primary_conn)
self.repl_set.stop()
def initOplogThread(self, namespace_set=[], dest_mapping={}):
self.docman = CommandLoggerDocManager()
self.docman.command_helper = CommandHelper(namespace_set, dest_mapping)
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(self.docman, ),
oplog_progress_dict=self.oplog_progress,
ns_set=namespace_set,
dest_mapping=dest_mapping,
collection_dump=False)
self.opman.start()
def test_command_helper(self):
# Databases cannot be merged
mapping = {'a.x': 'c.x', 'b.x': 'c.y'}
self.assertRaises(errors.MongoConnectorError, CommandHelper,
list(mapping), mapping)
mapping = {'a.x': 'b.x', 'a.y': 'c.y'}
helper = CommandHelper(list(mapping) + ['a.z'], mapping)
self.assertEqual(set(helper.map_db('a')), set(['a', 'b', 'c']))
self.assertEqual(helper.map_db('d'), [])
self.assertEqual(helper.map_namespace('a.x'), 'b.x')
self.assertEqual(helper.map_namespace('a.z'), 'a.z')
self.assertEqual(helper.map_namespace('d.x'), None)
self.assertEqual(helper.map_collection('a', 'x'), ('b', 'x'))
self.assertEqual(helper.map_collection('a', 'z'), ('a', 'z'))
self.assertEqual(helper.map_collection('d', 'x'), (None, None))
def test_create_collection(self):
self.initOplogThread()
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_create_collection_skipped(self):
self.initOplogThread(['test.test'])
pymongo.collection.Collection(self.primary_conn['test2'],
'test2',
create=True)
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'drop': 'test'})
def test_drop_database(self):
self.initOplogThread()
pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
self.primary_conn.drop_database('test')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'dropDatabase': 1})
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(self.primary_conn['test'],
'test',
create=True)
coll.rename('test2')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1].get('renameCollection'),
'test.test')
self.assertEqual(self.docman.commands[1].get('to'), 'test.test2')
class TestOplogManager(unittest.TestCase):
"""Defines all the testing methods, as well as a method that sets up the
cluster
"""
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None):
if include_ns is None:
include_ns = []
if exclude_ns is None:
exclude_ns = []
if dest_mapping is None:
dest_mapping = {}
# include_ns must not exist together with exclude_ns
# dest_mapping must exist together with include_ns
# those checks have been tested in test_config.py so we skip that here.
self.dest_mapping_stru = DestMapping(include_ns, exclude_ns,
dest_mapping)
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
ns_set=include_ns,
ex_ns_set=exclude_ns
)
def init_dbs(self):
# includedb1.* & includedb2.includecol1 are interested collections
self.primary_conn["includedb1"]["includecol1"].insert_many(
[{"idb1col1": i} for i in range(1, 3)])
self.primary_conn["includedb1"]["includecol2"].insert_many(
[{"idb1col2": i} for i in range(1, 3)])
self.primary_conn["includedb2"]["includecol1"].insert_many(
[{"idb2col1": i} for i in range(1, 3)])
# the others are not interested collections
self.primary_conn["includedb2"]["excludecol2"].insert_many(
[{"idb2col2": i} for i in range(1, 3)])
self.primary_conn["excludedb3"]["excludecol1"].insert_many(
[{"idb3col1": i} for i in range(1, 3)])
def tearDown(self):
try:
self.opman.join()
except RuntimeError:
pass # OplogThread may not have been started
for db in self.primary_conn.database_names():
if db != "local":
self.primary_conn.drop_database(db)
close_client(self.primary_conn)
self.repl_set.stop()
def test_get_oplog_cursor(self):
'''Test the get_oplog_cursor method'''
# Put something in the dbs
self.init_dbs()
# timestamp is None - all oplog entries excluding no-ops are returned.
# wildcard include case no impact the result
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
got_cursor = self.opman.get_oplog_cursor(None)
oplog_cursor = self.oplog_coll.find(
{'op': {'$ne': 'n'}})
self.assertNotEqual(got_cursor, None)
self.assertEqual(got_cursor.count(), oplog_cursor.count())
# wildcard exclude case no impact the result
self.reset_opman([], ["includedb2.excludecol2", "excludedb3.*"], {})
got_cursor = self.opman.get_oplog_cursor(None)
oplog_cursor = self.oplog_coll.find(
{'op': {'$ne': 'n'}})
self.assertNotEqual(got_cursor, None)
self.assertEqual(got_cursor.count(), oplog_cursor.count())
# earliest entry is the only one at/after timestamp
doc = {"ts": bson.Timestamp(1000, 0), "idb1col1": 1}
self.primary_conn["includedb1"]["includecol1"].insert_one(doc)
latest_timestamp = self.opman.get_last_oplog_timestamp()
cursor = self.opman.get_oplog_cursor(latest_timestamp)
self.assertNotEqual(cursor, None)
self.assertEqual(cursor.count(), 1)
next_entry_id = next(cursor)['o']['_id']
retrieved = self.primary_conn.includedb1.includecol1.find_one(
next_entry_id)
self.assertEqual(retrieved, doc)
# many entries before and after timestamp
self.primary_conn["includedb1"]["includecol1"].insert_many(
[{"idb1col1": i} for i in range(2, 1002)])
oplog_cursor = self.oplog_coll.find(
{'op': {'$ne': 'n'},
'ns': {'$not': re.compile(r'\.(system|\$cmd)')}},
sort=[("ts", pymongo.ASCENDING)])
# initial insert + 1000 more inserts
self.assertEqual(oplog_cursor.count(), 11 + 1000)
pivot = oplog_cursor.skip(400).limit(-1)[0]
goc_cursor = self.opman.get_oplog_cursor(pivot["ts"])
self.assertEqual(goc_cursor.count(), 11 + 1000 - 400)
def test_get_last_oplog_timestamp(self):
"""Test the get_last_oplog_timestamp method"""
# empty oplog case has been tested in test_oplog_manager.py,
# skip that here.
# Put something in the dbs
self.init_dbs()
# Test non-empty oplog
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["includedb1"]["includecol1"].insert_one({
"idb1col1": i + 500
})
oplog = self.primary_conn["local"]["oplog.rs"]
oplog = oplog.find(
{'op': {'$ne': 'n'}}).sort(
"$natural", pymongo.DESCENDING).limit(-1)[0]
self.assertEqual(self.opman.get_last_oplog_timestamp(),
oplog["ts"])
def test_dump_collection(self):
"""Test the dump_collection method
Cases:
1. no namespace set is set
2. include namespace set is set
3. exclude namespace set is set
empty oplog case has been tested in test_oplog_manager.py,
skip that here.
"""
# Put something in the dbs
self.init_dbs()
# no namespace set is set
self.reset_opman([], [], {})
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 10)
# include namespace set is set
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 6)
# exclude namespace set is set
self.reset_opman([], ["includedb2.excludecol2", "excludedb3.*"], {})
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 6)
def test_dump_collection_with_error(self):
"""Test the dump_collection method with invalid documents.
Cases:
1. non-empty oplog, continue_on_error=True, invalid documents
"""
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
# non-empty oplog, continue_on_error=True, invalid documents
self.opman.continue_on_error = True
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
docs = [{'a': i} for i in range(100)]
for i in range(50, 60):
docs[i]['_upsert_exception'] = True
self.primary_conn['includedb1']['includecol3'].insert_many(docs)
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
docs = self.opman.doc_managers[0]._search()
docs = list(filter(lambda doc: 'a' in doc, docs))
docs.sort(key=lambda doc: doc['a'])
self.assertEqual(len(docs), 90)
expected_a = itertools.chain(range(0, 50), range(60, 100))
for doc, correct_a in zip(docs, expected_a):
self.assertEqual(doc['a'], correct_a)
def test_init_cursor(self):
"""Test the init_cursor method
Cases:
1. no last checkpoint, no collection dump
2. no last checkpoint, collection dump ok and stuff to dump
3. no last checkpoint, nothing to dump, stuff in oplog
4. no last checkpoint, nothing to dump, nothing in oplog
5. no last checkpoint, no collection dump, stuff in oplog
6. last checkpoint exists
7. last checkpoint is behind
"""
# N.B. these sub-cases build off of each other and cannot be re-ordered
# without side-effects
self.reset_opman(["includedb1.*", "includedb2.includecol1"], [], {})
# No last checkpoint, no collection dump, nothing in oplog
# "change oplog collection" to put nothing in oplog
self.opman.oplog = self.primary_conn["includedb1"]["emptycollection"]
self.opman.collection_dump = False
self.assertTrue(all(doc['op'] == 'n'
for doc in self.opman.init_cursor()[0]))
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, nothing in oplog
self.opman.collection_dump = True
cursor, cursor_empty = self.opman.init_cursor()
self.assertEqual(cursor, None)
self.assertTrue(cursor_empty)
self.assertEqual(self.opman.checkpoint, None)
# No last checkpoint, empty collections, something in oplog
self.opman.oplog = self.primary_conn['local']['oplog.rs']
collection = self.primary_conn["includedb1"]["includecol1"]
collection.insert_one({"idb1col1": 1})
collection.delete_one({"idb1col1": 1})
time.sleep(3)
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_empty = self.opman.init_cursor()
self.assertFalse(cursor_empty)
self.assertEqual(self.opman.checkpoint, last_ts)
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[self.opman.replset_name],
last_ts)
# No last checkpoint, no collection dump, something in oplog
self.opman.oplog_progress = LockingDict()
self.opman.collection_dump = False
collection.insert_one({"idb1col1": 2})
last_ts = self.opman.get_last_oplog_timestamp()
cursor, cursor_empty = self.opman.init_cursor()
for doc in cursor:
last_doc = doc
self.assertEqual(last_doc['o']['idb1col1'], 2)
self.assertEqual(self.opman.checkpoint, last_ts)
# Last checkpoint exists
progress = LockingDict()
self.opman.oplog_progress = progress
for i in range(1000):
collection.insert_one({"idb1col1": i + 500})
entry = list(
self.primary_conn["local"]["oplog.rs"].find(skip=200, limit=-2))
progress.get_dict()[self.opman.replset_name] = entry[0]["ts"]
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_empty = self.opman.init_cursor()
self.assertEqual(next(cursor)["ts"], entry[1]["ts"])
self.assertEqual(self.opman.checkpoint, entry[0]["ts"])
with self.opman.oplog_progress as prog:
self.assertEqual(prog.get_dict()[self.opman.replset_name],
entry[0]["ts"])
# Last checkpoint is behind
progress = LockingDict()
progress.get_dict()[self.opman.replset_name] = bson.Timestamp(1, 0)
self.opman.oplog_progress = progress
self.opman.checkpoint = None
cursor, cursor_empty = self.opman.init_cursor()
self.assertTrue(cursor_empty)
self.assertEqual(cursor, None)
self.assertIsNotNone(self.opman.checkpoint)
def test_namespace_mapping(self):
"""Test mapping of namespaces
Cases:
upsert/delete/update of documents:
1. in namespace set, mapping provided
2. outside of namespace set, mapping provided
"""
source_ns_wildcard = ["includedb1.*", "includedb2.includecol1"]
source_ns = ["includedb1.includecol1",
"includedb1.includecol2",
"includedb2.includecol1"]
phony_ns = ["includedb2.excludecol2", "excludedb3.excludecol1"]
dest_mapping = {
"includedb1.*": "newdb1_*.bar",
"includedb2.includecol1": "newdb2.newcol1"
}
self.reset_opman(source_ns_wildcard, [], dest_mapping)
docman = self.opman.doc_managers[0]
dest_mapping_stru = self.opman.dest_mapping_stru
# start replicating
self.opman.start()
base_doc = {"_id": 1, "name": "superman"}
# doc in namespace set
for ns in source_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
assert_soon(lambda: len(docman._search()) == 1)
self.assertEqual(docman._search()[0]["ns"],
dest_mapping_stru.get(ns))
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1},
{"$set": {"weakness": "kryptonite"}}
)
def update_complete():
docs = docman._search()
for d in docs:
if d.get("weakness") == "kryptonite":
return True
return False
assert_soon(update_complete)
self.assertEqual(docman._search()[0]["ns"],
dest_mapping_stru.get(ns))
bad = [d for d in docman._search() if d["ns"] == ns]
self.assertEqual(len(bad), 0)
# test delete
self.primary_conn[db][coll].delete_one({"_id": 1})
assert_soon(lambda: len(docman._search()) == 0)
bad = [d for d in docman._search()
if d["ns"] == dest_mapping_stru.get(ns)]
self.assertEqual(len(bad), 0)
# cleanup
self.primary_conn[db][coll].delete_many({})
self.opman.doc_managers[0]._delete()
# doc not in namespace set
for ns in phony_ns:
db, coll = ns.split(".", 1)
# test insert
self.primary_conn[db][coll].insert_one(base_doc)
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
# test update
self.primary_conn[db][coll].update_one(
{"_id": 1},
{"$set": {"weakness": "kryptonite"}}
)
time.sleep(1)
self.assertEqual(len(docman._search()), 0)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
self.reset_opman(["includedb1.*"], [], {})
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["includedb1"]["includecol1"].insert_one({
"name": "kermit",
"color": "green"
})
self.primary_conn["includedb1"]["includecol2"].insert_one({
"name": "elmo",
"color": "firetruck red"
})
self.primary_conn["excludedb2"]["excludecol1"].insert_one({
"name": "panda",
"color": "white and black"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets"
)
self.primary_conn["includedb1"]["includecol2"].delete_one({
"name": "elmo"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets"
)
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
class TestCommandReplication(unittest.TestCase):
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_progress = LockingDict()
self.opman = None
def tearDown(self):
try:
if self.opman:
self.opman.join()
except RuntimeError:
pass
close_client(self.primary_conn)
self.repl_set.stop()
def initOplogThread(self, namespace_set=[], dest_mapping={}):
self.docman = CommandLoggerDocManager()
self.docman.command_helper = CommandHelper(namespace_set, dest_mapping)
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(self.docman,),
oplog_progress_dict=self.oplog_progress,
ns_set=namespace_set,
dest_mapping=dest_mapping,
collection_dump=False
)
self.opman.start()
def test_command_helper(self):
# Databases cannot be merged
mapping = {
'a.x': 'c.x',
'b.x': 'c.y'
}
self.assertRaises(errors.MongoConnectorError,
CommandHelper,
list(mapping), mapping)
mapping = {
'a.x': 'b.x',
'a.y': 'c.y'
}
helper = CommandHelper(list(mapping) + ['a.z'], mapping)
self.assertEqual(set(helper.map_db('a')), set(['a', 'b', 'c']))
self.assertEqual(helper.map_db('d'), [])
self.assertEqual(helper.map_namespace('a.x'), 'b.x')
self.assertEqual(helper.map_namespace('a.z'), 'a.z')
self.assertEqual(helper.map_namespace('d.x'), None)
self.assertEqual(helper.map_collection('a', 'x'), ('b', 'x'))
self.assertEqual(helper.map_collection('a', 'z'), ('a', 'z'))
self.assertEqual(helper.map_collection('d', 'x'), (None, None))
def test_create_collection(self):
self.initOplogThread()
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_create_collection_skipped(self):
self.initOplogThread(['test.test'])
pymongo.collection.Collection(
self.primary_conn['test2'], 'test2', create=True)
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
assert_soon(lambda: self.docman.commands)
self.assertEqual(len(self.docman.commands), 1)
self.assertEqual(self.docman.commands[0], {'create': 'test'})
def test_drop_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.drop()
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'drop': 'test'})
def test_drop_database(self):
self.initOplogThread()
pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
self.primary_conn.drop_database('test')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(self.docman.commands[1], {'dropDatabase': 1})
def test_rename_collection(self):
self.initOplogThread()
coll = pymongo.collection.Collection(
self.primary_conn['test'], 'test', create=True)
coll.rename('test2')
assert_soon(lambda: len(self.docman.commands) == 2)
self.assertEqual(
self.docman.commands[1].get('renameCollection'),
'test.test')
self.assertEqual(
self.docman.commands[1].get('to'),
'test.test2')
