def test_dump_collection(self):
"""Test dump_collection in oplog_manager. Assertion failure if it
doesn't pass
"""
test_oplog, primary_conn, search_ts = self.get_oplog_thread()
#with documents
primary_conn['test']['test'].insert({'name': 'paulie'})
search_ts = test_oplog.get_last_oplog_timestamp()
test_oplog.dump_collection()
doc_manager = test_oplog.doc_managers[0]
doc_manager.commit()
solr_results = doc_manager._search()
self.assertEqual(len(solr_results), 1)
solr_doc = solr_results[0]
self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts)
self.assertEqual(solr_doc['name'], 'paulie')
self.assertEqual(solr_doc['ns'], 'test.test')
# test multiple targets
doc_managers = [DocManager(), DocManager(), DocManager()]
test_oplog.doc_managers = doc_managers
primary_conn["test"]["test"].remove()
for i in range(1000):
primary_conn["test"]["test"].insert({"i": i})
test_oplog.dump_collection()
for dm in doc_managers:
self.assertEqual(len(dm._search()), 1000)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
opman, primary_conn, oplog_coll = self.get_oplog_thread()
doc_managers = [DocManager(), DocManager(), DocManager()]
opman.doc_managers = doc_managers
# start replicating
opman.start()
primary_conn["test"]["test"].insert({
"name": "kermit",
"color": "green"
})
primary_conn["test"]["test"].insert({
"name": "elmo",
"color": "firetruck red"
})
self.assertTrue(
wait_for(lambda: sum(len(d._search()) for d in doc_managers) == 6),
"OplogThread should be able to replicate to multiple targets")
primary_conn["test"]["test"].remove({"name": "elmo"})
self.assertTrue(
wait_for(lambda: sum(len(d._search()) for d in doc_managers) == 3),
"OplogThread should be able to replicate to multiple targets")
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
# cleanup
opman.join()
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["test"]["test"].insert_one({
"name": "kermit",
"color": "green"
})
self.primary_conn["test"]["test"].insert_one({
"name": "elmo",
"color": "firetruck red"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets")
self.primary_conn["test"]["test"].delete_one({"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets")
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_start_with_auth(self):
dm = DocManager()
connector = Connector(mongo_address=self.cluster.uri,
doc_managers=[dm],
auth_username=db_user,
auth_key=db_password)
connector.start()
# Insert some documents into the sharded cluster. These
# should go to the DocManager, and the connector should not
# have an auth failure.
self.cluster.client().test.test.insert_one({'auth_failure': False})
assert_soon(lambda: len(dm._search()) > 0)
connector.join()
def setUp(self):
# Create a new oplog progress file
try:
os.unlink("oplog.timestamp")
except OSError:
pass
open("oplog.timestamp", "w").close()
# Start a replica set
self.repl_set = ReplicaSet().start()
# Connection to the replica set as a whole
self.main_conn = self.repl_set.client()
# Connection to the primary specifically
self.primary_conn = self.repl_set.primary.client()
# Connection to the secondary specifically
self.secondary_conn = self.repl_set.secondary.client(
read_preference=ReadPreference.SECONDARY_PREFERRED)
# Wipe any test data
self.main_conn["test"]["mc"].drop()
# Oplog thread
doc_manager = DocManager()
oplog_progress = LockingDict()
self.opman = OplogThread(primary_client=self.main_conn,
doc_managers=(doc_manager, ),
oplog_progress_dict=oplog_progress,
ns_set=["test.mc"])
def setUp(self):
self.repl_set = ReplicaSet().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict())
def get_new_oplog(cls):
""" Set up connection with mongo. Returns oplog, the connection and
oplog collection
This function does not clear the oplog
"""
is_sharded = True
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"]))
if primary_conn['admin'].command("isMaster")['ismaster'] is False:
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"]))
mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN'])
if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]:
mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN'])
is_sharded = False
oplog_coll = primary_conn['local']['oplog.rs']
namespace_set = ['test.test']
doc_manager = DocManager()
oplog = OplogThread(primary_conn=primary_conn,
main_address=mongos_addr,
oplog_coll=oplog_coll,
is_sharded=is_sharded,
doc_manager=doc_manager,
oplog_progress_dict=LockingDict(),
namespace_set=namespace_set,
auth_key=cls.AUTH_KEY,
auth_username=AUTH_USERNAME,
repl_set="demo-repl")
return (oplog, primary_conn, oplog.main_connection, oplog_coll)
def get_oplog_thread(cls):
""" Set up connection with mongo.
Returns oplog, the connection and oplog collection.
This function clears the oplog.
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"]))
if primary_conn['admin'].command("isMaster")['ismaster'] is False:
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"]))
mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"])
mongos = Connection(mongos_addr)
mongos['alpha']['foo'].drop()
oplog_coll = primary_conn['local']['oplog.rs']
oplog_coll.drop() # reset the oplog
primary_conn['local'].create_collection('oplog.rs',
capped=True,
size=1000000)
namespace_set = ['test.test', 'alpha.foo']
doc_manager = DocManager()
oplog = OplogThread(primary_conn,
mongos_addr, oplog_coll, True, doc_manager,
LockingDict(), namespace_set, cls.AUTH_KEY,
AUTH_USERNAME)
return (oplog, primary_conn, oplog_coll, mongos)
def test_skipped_oplog_entry_updates_checkpoint(self):
repl_set = ReplicaSetSingle().start()
conn = repl_set.client()
opman = OplogThread(
primary_client=conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
namespace_config=NamespaceConfig(namespace_set=["test.test"]),
)
opman.start()
# Insert a document into an included collection
conn["test"]["test"].insert_one({"test": 1})
last_ts = opman.get_last_oplog_timestamp()
assert_soon(
lambda: last_ts == opman.checkpoint,
"OplogThread never updated checkpoint to non-skipped " "entry.",
)
self.assertEqual(len(opman.doc_managers[0]._search()), 1)
# Make sure that the oplog thread updates its checkpoint on every
# oplog entry.
conn["test"]["ignored"].insert_one({"test": 1})
last_ts = opman.get_last_oplog_timestamp()
assert_soon(
lambda: last_ts == opman.checkpoint,
"OplogThread never updated checkpoint to skipped entry.",
)
opman.join()
conn.close()
repl_set.stop()
def get_oplog_thread(cls):
""" Set up connection with mongo. Returns oplog, the connection and
oplog collection
This function clears the oplog
"""
is_sharded = True
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"]))
if primary_conn['admin'].command("isMaster")['ismaster'] is False:
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"]))
primary_conn['test']['test'].drop()
mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN'])
if PORTS_ONE["MAIN"] == PORTS_ONE["PRIMARY"]:
mongos_addr = "%s:%s" % (HOSTNAME, PORTS_ONE['MAIN'])
is_sharded = False
oplog_coll = primary_conn['local']['oplog.rs']
oplog_coll.drop() # reset the oplog
primary_conn['local'].create_collection('oplog.rs', capped=True,
size=1000000)
namespace_set = ['test.test']
doc_manager = DocManager()
oplog = OplogThread(primary_conn, mongos_addr, oplog_coll, is_sharded,
doc_manager, LockingDict(),
namespace_set, cls.AUTH_KEY, AUTH_USERNAME,
repl_set="demo-repl")
return(oplog, primary_conn, oplog_coll)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None):
self.namespace_config = NamespaceConfig(namespace_set=include_ns,
ex_namespace_set=exclude_ns,
namespace_options=dest_mapping)
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
namespace_config=self.namespace_config)
def test_start_with_auth(self):
dm = DocManager()
connector = Connector(
mongo_address=self.cluster.uri,
doc_managers=[dm],
auth_username=db_user,
auth_key=db_password
)
connector.start()
# Insert some documents into the sharded cluster. These
# should go to the DocManager, and the connector should not
# have an auth failure.
self.cluster.client().test.test.insert({'auth_failure': False})
assert_soon(lambda: len(dm._search()) > 0)
connector.join()
def setUp(self):
self.namespace_config = NamespaceConfig()
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
namespace_config=self.namespace_config,
)
def test_many_targets(self):
"""Test that one OplogThread is capable of replicating to more than
one target.
"""
self.reset_opman(["includedb1.*"], [], {})
doc_managers = [DocManager(), DocManager(), DocManager()]
self.opman.doc_managers = doc_managers
# start replicating
self.opman.start()
self.primary_conn["includedb1"]["includecol1"].insert_one({
"name":
"kermit",
"color":
"green"
})
self.primary_conn["includedb1"]["includecol2"].insert_one({
"name":
"elmo",
"color":
"firetruck red"
})
self.primary_conn["excludedb2"]["excludecol1"].insert_one({
"name":
"panda",
"color":
"white and black"
})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 6,
"OplogThread should be able to replicate to multiple targets",
)
self.primary_conn["includedb1"]["includecol2"].delete_one(
{"name": "elmo"})
assert_soon(
lambda: sum(len(d._search()) for d in doc_managers) == 3,
"OplogThread should be able to replicate to multiple targets",
)
for d in doc_managers:
self.assertEqual(d._search()[0]["name"], "kermit")
def test_fields_and_exclude(self):
fields = ['a', 'b', 'c', '_id']
exclude_fields = ['x', 'y', 'z']
# Test setting both to None in constructor
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
fields=None,
exclude_fields=None)
self._check_fields(opman, [], [], None)
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
fields=None,
exclude_fields=exclude_fields)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
# Test setting fields when exclude_fields is set
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"fields", fields)
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"fields", None)
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
exclude_fields=None,
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", exclude_fields)
self.assertRaises(errors.InvalidConfiguration, setattr, opman,
"exclude_fields", None)
self.assertRaises(errors.InvalidConfiguration,
OplogThread,
self.primary_conn, (DocManager(), ),
LockingDict(),
self.dest_mapping_stru,
fields=fields,
exclude_fields=exclude_fields)
def setUp(self):
if db_user and db_password:
auth_args = dict(auth_username=db_user, auth_key=db_password)
else:
auth_args = {}
self.cluster = ShardedClusterSingle().start()
self.dm = DocManager()
self.connector = Connector(mongo_address=self.cluster.uri,
doc_managers=[self.dm],
**auth_args)
self.connector.start()
def setUp(self):
self.repl_set = ReplicaSetSingle().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.dest_mapping_stru = DestMapping([], [], {})
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
)
def setUp(self):
self.repl_set = ReplicaSetSingle().start()
self.primary_conn = self.repl_set.client()
self.oplog_coll = self.primary_conn.local["oplog.rs"]
self.opman = OplogThread(
primary_client=self.primary_conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
namespace_config=NamespaceConfig(
namespace_options={"test.*": True, "gridfs.*": {"gridfs": True}}
),
)
def test_dump_collection(self):
"""Test the dump_collection method
Cases:
1. empty oplog
2. non-empty oplog, with gridfs collections
3. non-empty oplog, specified a namespace-set, none of the oplog
entries are for collections in the namespace-set
"""
# Test with empty oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
last_ts = self.opman.dump_collection()
self.assertEqual(last_ts, None)
# Test with non-empty oplog with gridfs collections
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
# Insert 10 gridfs files
for i in range(10):
fs = gridfs.GridFS(self.primary_conn["gridfs"],
collection="test" + str(i))
fs.put(b"hello world")
# Insert 1000 documents
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({
"i": i + 500
})
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 1010)
# Case 3
# 1MB oplog so that we can rollover quickly
repl_set = ReplicaSetSingle(oplogSize=1).start()
conn = repl_set.client()
opman = OplogThread(
primary_client=conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
namespace_config=NamespaceConfig(namespace_set=["test.test"]),
)
# Insert a document into an included collection
conn["test"]["test"].insert_one({"test": 1})
# Cause the oplog to rollover on a non-included collection
while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}):
conn["test"]["ignored"].insert_many(
[{"test": "1" * 1024} for _ in range(1024)])
last_ts = opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, opman.dump_collection())
self.assertEqual(len(opman.doc_managers[0]._search()), 1)
conn.close()
repl_set.stop()
def test_dump_collection(self):
"""Test dump_collection in oplog_manager. Assertion failure if it
doesn't pass
"""
test_oplog, primary_conn, search_ts = self.get_oplog_thread()
solr = DocManager()
test_oplog.doc_manager = solr
# with documents
primary_conn["test"]["test"].insert({"name": "paulie"})
search_ts = test_oplog.get_last_oplog_timestamp()
test_oplog.dump_collection()
test_oplog.doc_manager.commit()
solr_results = solr._search()
self.assertEqual(len(solr_results), 1)
solr_doc = solr_results[0]
self.assertEqual(long_to_bson_ts(solr_doc["_ts"]), search_ts)
self.assertEqual(solr_doc["name"], "paulie")
self.assertEqual(solr_doc["ns"], "test.test")
def test_dump_collection(self):
"""Test dump_collection in oplog_manager. Assertion failure if it
doesn't pass
"""
test_oplog, primary_conn, search_ts = self.get_oplog_thread()
solr = DocManager()
test_oplog.doc_manager = solr
#with documents
primary_conn['test']['test'].insert({'name': 'paulie'})
search_ts = test_oplog.get_last_oplog_timestamp()
test_oplog.dump_collection()
test_oplog.doc_manager.commit()
solr_results = solr._search()
self.assertEqual(len(solr_results), 1)
solr_doc = solr_results[0]
self.assertEqual(long_to_bson_ts(solr_doc['_ts']), search_ts)
self.assertEqual(solr_doc['name'], 'paulie')
self.assertEqual(solr_doc['ns'], 'test.test')
def test_dump_collection(self):
"""Test dump_collection in oplog_manager.
Assertion failure if it doesn't pass
"""
test_oplog, search_ts, solr, mongos = self.get_oplog_thread()
solr = DocManager()
test_oplog.doc_manager = solr
# with documents
safe_mongo_op(mongos['alpha']['foo'].insert, {'name': 'paulie'})
search_ts = test_oplog.get_last_oplog_timestamp()
test_oplog.dump_collection()
test_oplog.doc_manager.commit()
solr_results = solr._search()
assert (len(solr_results) == 1)
solr_doc = solr_results[0]
assert (long_to_bson_ts(solr_doc['_ts']) == search_ts)
assert (solr_doc['name'] == 'paulie')
assert (solr_doc['ns'] == 'alpha.foo')
def test_dump_collection(self):
"""Test dump_collection in oplog_manager.
Assertion failure if it doesn't pass
"""
test_oplog, search_ts, solr, mongos = self.get_oplog_thread()
solr = DocManager()
test_oplog.doc_manager = solr
# with documents
safe_mongo_op(mongos['alpha']['foo'].insert, {'name': 'paulie'})
search_ts = test_oplog.get_last_oplog_timestamp()
test_oplog.dump_collection()
test_oplog.doc_manager.commit()
solr_results = solr._search()
assert (len(solr_results) == 1)
solr_doc = solr_results[0]
assert (long_to_bson_ts(solr_doc['_ts']) == search_ts)
assert (solr_doc['name'] == 'paulie')
assert (solr_doc['ns'] == 'alpha.foo')
def setUpClass(cls):
if db_user and db_password:
auth_args = dict(auth_username=db_user, auth_key=db_password)
else:
auth_args = {}
cls.cluster = ShardedClusterSingle().start()
cls.main_uri = cls.cluster.uri + '/?readPreference=primaryPreferred'
cls.dm = DocManager()
cls.connector = Connector(mongo_address=cls.main_uri,
doc_managers=[cls.dm],
**auth_args)
cls.connector.start()
assert_soon(lambda: len(cls.connector.shard_set) == 2,
message='connector failed to find both shards!')
def test_dump_collection(self):
"""Test the dump_collection method
Cases:
1. empty oplog
2. non-empty oplog
3. non-empty oplog, specified a namespace-set, none of the oplog
entries are for collections in the namespace-set
"""
# Test with empty oplog
self.opman.oplog = self.primary_conn["test"]["emptycollection"]
last_ts = self.opman.dump_collection()
self.assertEqual(last_ts, None)
# Test with non-empty oplog
self.opman.oplog = self.primary_conn["local"]["oplog.rs"]
for i in range(1000):
self.primary_conn["test"]["test"].insert_one({
"i": i + 500
})
last_ts = self.opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, self.opman.dump_collection())
self.assertEqual(len(self.opman.doc_managers[0]._search()), 1000)
# Case 3
# 1MB oplog so that we can rollover quickly
repl_set = ReplicaSetSingle(oplogSize=1).start()
conn = repl_set.client()
dest_mapping_stru = DestMapping(["test.test"], [], {})
opman = OplogThread(
primary_client=conn,
doc_managers=(DocManager(),),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=dest_mapping_stru,
ns_set=set(["test.test"])
)
# Insert a document into a ns_set collection
conn["test"]["test"].insert_one({"test": 1})
# Cause the oplog to rollover on a non-ns_set collection
while conn["local"]["oplog.rs"].find_one({"ns": "test.test"}):
conn["test"]["ignored"].insert_many(
[{"test": "1" * 1024} for _ in range(1024)])
last_ts = opman.get_last_oplog_timestamp()
self.assertEqual(last_ts, opman.dump_collection())
self.assertEqual(len(opman.doc_managers[0]._search()), 1)
conn.close()
repl_set.stop()
def setUp(self):
_, _, self.primary_p = start_replica_set('test-oplog-manager')
self.primary_conn = pymongo.MongoClient(mongo_host, self.primary_p)
self.oplog_coll = self.primary_conn.local['oplog.rs']
self.opman = OplogThread(primary_conn=self.primary_conn,
main_address='%s:%d' %
(mongo_host, self.primary_p),
oplog_coll=self.oplog_coll,
is_sharded=False,
doc_manager=DocManager(),
oplog_progress_dict=LockingDict(),
namespace_set=None,
auth_key=None,
auth_username=None,
repl_set='test-oplog-manager')
def test_init_cursor(self):
"""Test init_cursor in oplog_manager. Assertion failure if it
doesn't pass
"""
test_oplog, primary_conn, search_ts = self.get_oplog_thread()
test_oplog.checkpoint = None # needed for these tests
# initial tests with no config file and empty oplog
self.assertEqual(test_oplog.init_cursor(), None)
# no config, single oplog entry
primary_conn["test"]["test"].insert({"name": "paulie"})
search_ts = test_oplog.get_last_oplog_timestamp()
cursor = test_oplog.init_cursor()
self.assertEqual(cursor.count(), 1)
self.assertEqual(test_oplog.checkpoint, search_ts)
# with config file, assert that size != 0
os.system("touch temp_config.txt")
cursor = test_oplog.init_cursor()
oplog_dict = test_oplog.oplog_progress.get_dict()
self.assertEqual(cursor.count(), 1)
self.assertTrue(str(test_oplog.oplog) in oplog_dict)
self.assertTrue(oplog_dict[str(test_oplog.oplog)] == test_oplog.checkpoint)
os.system("rm temp_config.txt")
# test init_cursor when OplogThread created with/without no-dump option
# insert some documents (will need to be dumped)
primary_conn["test"]["test"].remove()
primary_conn["test"]["test"].insert(({"_id": i} for i in range(100)))
# test no-dump option
docman = DocManager()
docman._delete()
test_oplog.doc_manager = docman
test_oplog.collection_dump = False
test_oplog.oplog_progress = LockingDict()
# init_cursor has the side-effect of causing a collection dump
test_oplog.init_cursor()
self.assertEqual(len(docman._search()), 0)
# test w/o no-dump option
docman._delete()
test_oplog.collection_dump = True
test_oplog.oplog_progress = LockingDict()
test_oplog.init_cursor()
self.assertEqual(len(docman._search()), 100)
def reset_opman(self, include_ns=None, exclude_ns=None, dest_mapping=None):
if include_ns is None:
include_ns = []
if exclude_ns is None:
exclude_ns = []
if dest_mapping is None:
dest_mapping = {}
# include_ns must not exist together with exclude_ns
# dest_mapping must exist together with include_ns
# those checks have been tested in test_config.py so we skip that here.
self.dest_mapping_stru = DestMapping(include_ns, exclude_ns,
dest_mapping)
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
ns_set=include_ns,
ex_ns_set=exclude_ns)
def get_new_oplog(cls):
""" Set up connection with mongo.
Returns oplog, the connection and oplog collection
This function does not clear the oplog
"""
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["PRIMARY"]))
if primary_conn['admin'].command("isMaster")['ismaster'] is False:
primary_conn = Connection(HOSTNAME, int(PORTS_ONE["SECONDARY"]))
mongos = "%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"])
oplog_coll = primary_conn['local']['oplog.rs']
namespace_set = ['test.test', 'alpha.foo']
doc_manager = DocManager()
oplog = OplogThread(primary_conn, mongos, oplog_coll, True,
doc_manager, LockingDict(), namespace_set,
cls.AUTH_KEY, AUTH_USERNAME)
return (oplog, primary_conn, oplog_coll, oplog.main_connection)
def setUp(self):
# Create a new oplog progress file
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
# Start a replica set
_, self.secondary_p, self.primary_p = start_replica_set('rollbacks')
# Connection to the replica set as a whole
self.main_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p),
replicaSet='rollbacks')
# Connection to the primary specifically
self.primary_conn = MongoClient('%s:%d' % (mongo_host, self.primary_p))
# Connection to the secondary specifically
self.secondary_conn = MongoClient(
'%s:%d' % (mongo_host, self.secondary_p),
read_preference=ReadPreference.SECONDARY_PREFERRED
)
# Wipe any test data
self.main_conn["test"]["mc"].drop()
# Oplog thread
doc_manager = DocManager()
oplog_progress = LockingDict()
self.opman = OplogThread(
primary_conn=self.main_conn,
main_address='%s:%d' % (mongo_host, self.primary_p),
oplog_coll=self.main_conn["local"]["oplog.rs"],
is_sharded=False,
doc_manager=doc_manager,
oplog_progress_dict=oplog_progress,
namespace_set=["test.mc"],
auth_key=None,
auth_username=None,
repl_set="rollbacks"
)
def setUp(self):
# Create a new oplog progress file
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
# Start a replica set
start_cluster(sharded=False, use_mongos=False)
# Connection to the replica set as a whole
self.main_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"],
replicaSet="demo-repl")
# Connection to the primary specifically
self.primary_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"])
# Connection to the secondary specifically
self.secondary_conn = Connection(
"localhost:%s" % PORTS_ONE["SECONDARY"],
read_preference=ReadPreference.SECONDARY_PREFERRED)
# Wipe any test data
self.main_conn["test"]["mc"].drop()
# Oplog thread
doc_manager = DocManager()
oplog_progress = LockingDict()
self.opman = OplogThread(
primary_conn=self.main_conn,
main_address="localhost:%s" % PORTS_ONE["PRIMARY"],
oplog_coll=self.main_conn["local"]["oplog.rs"],
is_sharded=False,
doc_manager=doc_manager,
oplog_progress_dict=oplog_progress,
namespace_set=["test.mc"],
auth_key=None,
auth_username=None,
repl_set="demo-repl")
def test_rollback(self):
"""Test rollback in oplog_manager. Assertion failure if it doesn't pass
We force a rollback by inserting a doc, killing the primary,
inserting another doc, killing the new primary, and then restarting
both.
"""
os.system('rm config.txt; touch config.txt')
test_oplog, primary_conn, mongos, solr = self.get_new_oplog()
if not start_cluster():
self.fail('Cluster could not be started successfully!')
solr = DocManager()
test_oplog.doc_manager = solr
solr._delete() # equivalent to solr.delete(q='*: *')
mongos['test']['test'].remove({})
mongos['test']['test'].insert(
{'_id': ObjectId('4ff74db3f646462b38000001'),
'name': 'paulie'},
safe=True
)
while (mongos['test']['test'].find().count() != 1):
time.sleep(1)
cutoff_ts = test_oplog.get_last_oplog_timestamp()
first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts),
'ns': 'test.test',
'_id': ObjectId('4ff74db3f646462b38000001')}
#try kill one, try restarting
kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin = new_primary_conn['admin']
while admin.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
mongos['test']['test'].insert({
'_id': ObjectId('4ff74db3f646462b38000002'),
'name': 'paul'},
safe=True)
break
except OperationFailure:
count += 1
if count > 60:
self.fail('Call to insert doc failed too many times')
time.sleep(1)
continue
while (mongos['test']['test'].find().count() != 2):
time.sleep(1)
kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
#wait for master to be established
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
#wait for secondary to be established
admin = new_primary_conn['admin']
while admin.command("replSetGetStatus")['myState'] != 2:
time.sleep(1)
while retry_until_ok(mongos['test']['test'].find().count) != 1:
time.sleep(1)
self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY'])
self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY'])
last_ts = test_oplog.get_last_oplog_timestamp()
second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts),
'ns': 'test.test',
'_id': ObjectId('4ff74db3f646462b38000002')}
test_oplog.doc_manager.upsert(first_doc)
test_oplog.doc_manager.upsert(second_doc)
test_oplog.rollback()
test_oplog.doc_manager.commit()
results = solr._search()
assert(len(results) == 1)
self.assertEqual(results[0]['name'], 'paulie')
self.assertTrue(results[0]['_ts'] <= bson_ts_to_long(cutoff_ts))
def test_rollback(self):
"""Test rollback in oplog_manager. Assertion failure if it doesn't pass
We force a rollback by inserting a doc, killing primary, inserting
another doc, killing the new primary, and then restarting both
servers.
"""
os.system('rm %s; touch %s' % (CONFIG, CONFIG))
if not start_cluster(sharded=True):
self.fail("Shards cannot be added to mongos")
test_oplog, primary_conn, solr, mongos = self.get_new_oplog()
solr = DocManager()
test_oplog.doc_manager = solr
solr._delete() # equivalent to solr.delete(q='*:*')
safe_mongo_op(mongos['alpha']['foo'].remove, {})
safe_mongo_op(mongos['alpha']['foo'].insert,
{'_id': ObjectId('4ff74db3f646462b38000001'),
'name': 'paulie'})
cutoff_ts = test_oplog.get_last_oplog_timestamp()
obj2 = ObjectId('4ff74db3f646462b38000002')
first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts),
'ns': 'alpha.foo',
'_id': ObjectId('4ff74db3f646462b38000001')}
# try kill one, try restarting
kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY'])
new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY']))
admin_db = new_primary_conn['admin']
while admin_db.command("isMaster")['ismaster'] is False:
time.sleep(1)
time.sleep(5)
count = 0
while True:
try:
mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'})
break
except OperationFailure:
time.sleep(1)
count += 1
if count > 60:
self.fail('Insert failed too many times in rollback')
continue
kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY'])
start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a",
"/replset1a.log", None)
# wait for master to be established
while primary_conn['admin'].command("isMaster")['ismaster'] is False:
time.sleep(1)
start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b",
"/replset1b.log", None)
# wait for secondary to be established
admin_db = new_primary_conn['admin']
while admin_db.command("replSetGetStatus")['myState'] != 2:
time.sleep(1)
while retry_until_ok(mongos['alpha']['foo'].find().count) != 1:
time.sleep(1)
self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY'])
self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY'])
last_ts = test_oplog.get_last_oplog_timestamp()
second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts),
'ns': 'alpha.foo', '_id': obj2}
test_oplog.doc_manager.upsert(first_doc)
test_oplog.doc_manager.upsert(second_doc)
test_oplog.rollback()
test_oplog.doc_manager.commit()
results = solr._search()
self.assertEqual(len(results), 1)
results_doc = results[0]
self.assertEqual(results_doc['name'], 'paulie')
self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def setUp(self):
self.dest_mapping_stru = DestMapping([], [], {})
self.opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru)
def test_exclude_fields_constructor(self):
# Test with the "_id" field in exclude_fields
exclude_fields = ["_id", "title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
exclude_fields=exclude_fields)
exclude_fields.remove('_id')
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in ['extra1', 'extra2']), filtered)
# Test without "_id" field included in exclude_fields
exclude_fields = ["title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
exclude_fields=exclude_fields)
self._check_fields(opman, [], exclude_fields,
dict((f, 0) for f in exclude_fields))
extra_fields = extra_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'extra1': 1, 'extra2': 1}, filtered)
# Test with only "_id" field in exclude_fields
exclude_fields = ["_id"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
exclude_fields=exclude_fields)
self._check_fields(opman, [], [], None)
extra_fields = exclude_fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
# Test with nothing set for exclude_fields
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
exclude_fields=None)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
def test_fields_constructor(self):
# Test with "_id" field in constructor
fields = ["_id", "title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test without "_id" field in constructor
fields = ["title", "content", "author"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
fields=fields)
fields.append('_id')
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in fields), filtered)
# Test with only "_id" field
fields = ["_id"]
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru,
fields=fields)
self._check_fields(opman, fields, [], dict((f, 1) for f in fields))
extra_fields = fields + ['extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual({'_id': 1}, filtered)
# Test with no fields set
opman = OplogThread(primary_client=self.primary_conn,
doc_managers=(DocManager(), ),
oplog_progress_dict=LockingDict(),
dest_mapping_stru=self.dest_mapping_stru)
self._check_fields(opman, [], [], None)
extra_fields = ['_id', 'extra1', 'extra2']
filtered = opman.filter_oplog_entry({
'op':
'i',
'o':
dict((f, 1) for f in extra_fields)
})['o']
self.assertEqual(dict((f, 1) for f in extra_fields), filtered)
