def setUpClass(cls): """ Initializes the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) use_mongos = True if PORTS_ONE['MONGOS'] != "27217": use_mongos = False cls.flag = start_cluster(use_mongos=use_mongos) if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") timer = Timer(60, abort_test) cls.connector = Connector( address="%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), oplog_checkpoint=CONFIG, target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None ) cls.synchronizer = cls.connector.doc_managers[0] timer.start() cls.connector.start() while len(cls.connector.shard_set) == 0: pass timer.cancel()
def setUpClass(cls): """ Initializes the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) use_mongos = True if PORTS_ONE['MONGOS'] != "27217": use_mongos = False cls.flag = start_cluster(use_mongos=use_mongos) if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") timer = Timer(60, abort_test) cls.connector = Connector(address="%s:%s" % (HOSTNAME, PORTS_ONE["MONGOS"]), oplog_checkpoint=CONFIG, target_url=None, ns_set=['test.test'], u_key='_id', auth_key=None) cls.synchronizer = cls.connector.doc_managers[0] timer.start() cls.connector.start() while len(cls.connector.shard_set) == 0: pass timer.cancel()
def setUpClass(cls): """ Initializes the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) use_mongos = True if MAIN_ADDRESS.split(":")[1] != "27217": use_mongos = False cls.flag = start_cluster(use_mongos=use_mongos)
def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) start_single_mongod_instance("30000", "/MC", "MC_log") cls.mongo_doc = DocManager("localhost:30000") cls.mongo_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection("%s:%s" % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl")
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set start_cluster(sharded=False, use_mongos=False) # Connection to the replica set as a whole self.main_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"], replicaSet="demo-repl") # Connection to the primary specifically self.primary_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"]) # Connection to the secondary specifically self.secondary_conn = Connection( "localhost:%s" % PORTS_ONE["SECONDARY"], read_preference=ReadPreference.SECONDARY_PREFERRED ) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address="localhost:%s" % PORTS_ONE["PRIMARY"], oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="demo-repl" )
def setUp(self): # Create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Start a replica set start_cluster(sharded=False, use_mongos=False) # Connection to the replica set as a whole self.main_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"], replicaSet="demo-repl") # Connection to the primary specifically self.primary_conn = Connection("localhost:%s" % PORTS_ONE["PRIMARY"]) # Connection to the secondary specifically self.secondary_conn = Connection( "localhost:%s" % PORTS_ONE["SECONDARY"], read_preference=ReadPreference.SECONDARY_PREFERRED) # Wipe any test data self.main_conn["test"]["mc"].drop() # Oplog thread doc_manager = DocManager() oplog_progress = LockingDict() self.opman = OplogThread( primary_conn=self.main_conn, main_address="localhost:%s" % PORTS_ONE["PRIMARY"], oplog_coll=self.main_conn["local"]["oplog.rs"], is_sharded=False, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mc"], auth_key=None, auth_username=None, repl_set="demo-repl")
def setUpClass(cls): """ Starts the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.elastic_doc = DocManager('localhost:9200') cls.elastic_doc._remove() cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MONGOS']), replicaSet="demo-repl") import logging logger = logging.getLogger() loglevel = logging.INFO logger.setLevel(loglevel)
def setUpClass(cls): os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.flag = start_cluster() if cls.flag: cls.conn = Connection('%s:%s' % (HOSTNAME, PORTS_ONE['MAIN']), replicaSet="demo-repl") # Creating a Solr object with an invalid URL # doesn't create an exception cls.solr_conn = Solr('http://localhost:8983/solr') try: cls.solr_conn.commit() except (SolrError, MissingSchema): cls.err_msg = "Cannot connect to Solr!" cls.flag = False if cls.flag: cls.solr_conn.delete(q='*:*') else: cls.err_msg = "Shards cannot be added to mongos"
def setUpClass(cls): """ Initializes the cluster """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) cls.AUTH_KEY = None cls.flag = True if AUTH_FILE: # We want to get the credentials from this file try: key = (open(AUTH_FILE)).read() re.sub(r'\s', '', key) cls.AUTH_KEY = key except IOError: print('Could not parse authentication file!') cls.flag = False cls.err_msg = "Could not read key file!" if not start_cluster(key_file=AUTH_FILE): cls.flag = False cls.err_msg = "Shards cannot be added to mongos"
def setUp(self): """ Initialize the cluster: Clean out the databases used by the tests Make connections to mongos, mongods Create and shard test collections Create OplogThreads """ # Start the cluster with a mongos on port 27217 self.mongos_p = start_cluster() # Connection to mongos mongos_address = '%s:%d' % (mongo_host, self.mongos_p) self.mongos_conn = MongoClient(mongos_address) # Connections to the shards shard1_ports = get_shard(self.mongos_p, 0) shard2_ports = get_shard(self.mongos_p, 1) self.shard1_prim_p = shard1_ports['primary'] self.shard1_scnd_p = shard1_ports['secondaries'][0] self.shard2_prim_p = shard2_ports['primary'] self.shard2_scnd_p = shard2_ports['secondaries'][0] self.shard1_conn = MongoClient('%s:%d' % (mongo_host, self.shard1_prim_p), replicaSet="demo-set-0") self.shard2_conn = MongoClient('%s:%d' % (mongo_host, self.shard2_prim_p), replicaSet="demo-set-1") self.shard1_secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.shard1_scnd_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) self.shard2_secondary_conn = MongoClient( '%s:%d' % (mongo_host, self.shard2_scnd_p), read_preference=ReadPreference.SECONDARY_PREFERRED ) # Wipe any test data self.mongos_conn["test"]["mcsharded"].drop() # Create and shard the collection test.mcsharded on the "i" field self.mongos_conn["test"]["mcsharded"].ensure_index("i") self.mongos_conn.admin.command("enableSharding", "test") self.mongos_conn.admin.command("shardCollection", "test.mcsharded", key={"i": 1}) # Pre-split the collection so that: # i < 1000 lives on shard1 # i >= 1000 lives on shard2 self.mongos_conn.admin.command(bson.SON([ ("split", "test.mcsharded"), ("middle", {"i": 1000}) ])) # disable the balancer self.mongos_conn.config.settings.update( {"_id": "balancer"}, {"$set": {"stopped": True}}, upsert=True ) # Move chunks to their proper places try: self.mongos_conn["admin"].command( "moveChunk", "test.mcsharded", find={"i": 1}, to="demo-set-0" ) except pymongo.errors.OperationFailure: pass # chunk may already be on the correct shard try: self.mongos_conn["admin"].command( "moveChunk", "test.mcsharded", find={"i": 1000}, to="demo-set-1" ) except pymongo.errors.OperationFailure: pass # chunk may already be on the correct shard # Make sure chunks are distributed correctly self.mongos_conn["test"]["mcsharded"].insert({"i": 1}) self.mongos_conn["test"]["mcsharded"].insert({"i": 1000}) def chunks_moved(): doc1 = self.shard1_conn.test.mcsharded.find_one() doc2 = self.shard2_conn.test.mcsharded.find_one() if None in (doc1, doc2): return False return doc1['i'] == 1 and doc2['i'] == 1000 assert_soon(chunks_moved) self.mongos_conn.test.mcsharded.remove() # create a new oplog progress file try: os.unlink("config.txt") except OSError: pass open("config.txt", "w").close() # Oplog threads (oplog manager) for each shard doc_manager = DocManager() oplog_progress = LockingDict() self.opman1 = OplogThread( primary_conn=self.shard1_conn, main_address='%s:%d' % (mongo_host, self.mongos_p), oplog_coll=self.shard1_conn["local"]["oplog.rs"], is_sharded=True, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], auth_key=None, auth_username=None ) self.opman2 = OplogThread( primary_conn=self.shard2_conn, main_address='%s:%d' % (mongo_host, self.mongos_p), oplog_coll=self.shard2_conn["local"]["oplog.rs"], is_sharded=True, doc_manager=doc_manager, oplog_progress_dict=oplog_progress, namespace_set=["test.mcsharded", "test.mcunsharded"], auth_key=None, auth_username=None )
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing primary, inserting another doc, killing the new primary, and then restarting both servers. """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) if not start_cluster(sharded=True): self.fail("Shards cannot be added to mongos") test_oplog, primary_conn, solr, mongos = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*:*') safe_mongo_op(mongos['alpha']['foo'].remove, {}) safe_mongo_op(mongos['alpha']['foo'].insert, { '_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie' }) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = { 'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'alpha.foo', '_id': ObjectId('4ff74db3f646462b38000001') } # try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'}) break except OperationFailure: time.sleep(1) count += 1 if count > 60: self.fail('Insert failed too many times in rollback') continue kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) # wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) # wait for secondary to be established admin_db = new_primary_conn['admin'] while admin_db.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['alpha']['foo'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = { 'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'alpha.foo', '_id': obj2 } test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() self.assertEqual(len(results), 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing primary, inserting another doc, killing the new primary, and then restarting both servers. """ os.system('rm %s; touch %s' % (CONFIG, CONFIG)) if not start_cluster(sharded=True): self.fail("Shards cannot be added to mongos") test_oplog, primary_conn, solr, mongos = self.get_new_oplog() solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*:*') safe_mongo_op(mongos['alpha']['foo'].remove, {}) safe_mongo_op(mongos['alpha']['foo'].insert, {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}) cutoff_ts = test_oplog.get_last_oplog_timestamp() obj2 = ObjectId('4ff74db3f646462b38000002') first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'alpha.foo', '_id': ObjectId('4ff74db3f646462b38000001')} # try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin_db = new_primary_conn['admin'] while admin_db.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['alpha']['foo'].insert({'_id': obj2, 'name': 'paul'}) break except OperationFailure: time.sleep(1) count += 1 if count > 60: self.fail('Insert failed too many times in rollback') continue kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) # wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) # wait for secondary to be established admin_db = new_primary_conn['admin'] while admin_db.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['alpha']['foo'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'alpha.foo', '_id': obj2} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() self.assertEqual(len(results), 1) results_doc = results[0] self.assertEqual(results_doc['name'], 'paulie') self.assertTrue(results_doc['_ts'] <= bson_ts_to_long(cutoff_ts))
def test_rollback(self): """Test rollback in oplog_manager. Assertion failure if it doesn't pass We force a rollback by inserting a doc, killing the primary, inserting another doc, killing the new primary, and then restarting both. """ os.system('rm config.txt; touch config.txt') test_oplog, primary_conn, mongos, solr = self.get_new_oplog() if not start_cluster(): self.fail('Cluster could not be started successfully!') solr = DocManager() test_oplog.doc_manager = solr solr._delete() # equivalent to solr.delete(q='*: *') mongos['test']['test'].remove({}) mongos['test']['test'].insert( {'_id': ObjectId('4ff74db3f646462b38000001'), 'name': 'paulie'}, safe=True ) while (mongos['test']['test'].find().count() != 1): time.sleep(1) cutoff_ts = test_oplog.get_last_oplog_timestamp() first_doc = {'name': 'paulie', '_ts': bson_ts_to_long(cutoff_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000001')} #try kill one, try restarting kill_mongo_proc(primary_conn.host, PORTS_ONE['PRIMARY']) new_primary_conn = Connection(HOSTNAME, int(PORTS_ONE['SECONDARY'])) admin = new_primary_conn['admin'] while admin.command("isMaster")['ismaster'] is False: time.sleep(1) time.sleep(5) count = 0 while True: try: mongos['test']['test'].insert({ '_id': ObjectId('4ff74db3f646462b38000002'), 'name': 'paul'}, safe=True) break except OperationFailure: count += 1 if count > 60: self.fail('Call to insert doc failed too many times') time.sleep(1) continue while (mongos['test']['test'].find().count() != 2): time.sleep(1) kill_mongo_proc(primary_conn.host, PORTS_ONE['SECONDARY']) start_mongo_proc(PORTS_ONE['PRIMARY'], "demo-repl", "/replset1a", "/replset1a.log", None) #wait for master to be established while primary_conn['admin'].command("isMaster")['ismaster'] is False: time.sleep(1) start_mongo_proc(PORTS_ONE['SECONDARY'], "demo-repl", "/replset1b", "/replset1b.log", None) #wait for secondary to be established admin = new_primary_conn['admin'] while admin.command("replSetGetStatus")['myState'] != 2: time.sleep(1) while retry_until_ok(mongos['test']['test'].find().count) != 1: time.sleep(1) self.assertEqual(str(new_primary_conn.port), PORTS_ONE['SECONDARY']) self.assertEqual(str(primary_conn.port), PORTS_ONE['PRIMARY']) last_ts = test_oplog.get_last_oplog_timestamp() second_doc = {'name': 'paul', '_ts': bson_ts_to_long(last_ts), 'ns': 'test.test', '_id': ObjectId('4ff74db3f646462b38000002')} test_oplog.doc_manager.upsert(first_doc) test_oplog.doc_manager.upsert(second_doc) test_oplog.rollback() test_oplog.doc_manager.commit() results = solr._search() assert(len(results) == 1) self.assertEqual(results[0]['name'], 'paulie') self.assertTrue(results[0]['_ts'] <= bson_ts_to_long(cutoff_ts))