def setUp(self): members = [ {}, { 'priority': 0 }, { 'priority': 0 }, ] res = ha_tools.start_replica_set(members, auth=True) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth try: self.c.admin.add_user('admin', 'adminpass') except: # SERVER-4225 pass self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout()
def test_passive_and_hidden(self): self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = self.c.pymongo_test w = len(self.c.secondaries) + 1 db.test.remove({}, safe=True, w=w) db.test.insert({'foo': 'bar'}, safe=True, w=w) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] hidden = ha_tools.get_hidden_members() hidden = [_partition_node(member) for member in hidden] self.assertEqual(self.c.secondaries, set(passives)) for mode in (ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED): db.read_preference = mode for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in passives) self.assertTrue(cursor._Cursor__connection_id not in hidden) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) db.read_preference = ReadPreference.SECONDARY_PREFERRED for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
class TestPassiveAndHidden(unittest.TestCase): def setUp(self): members = [{}, {'priority': 0}, {'arbiterOnly': True}, {'priority': 0, 'hidden': True}, {'priority': 0, 'slaveDelay': 5} ] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED) def tearDown(self): self.c.close() ha_tools.kill_all_members()
class TestMonitorRemovesRecoveringMember(unittest.TestCase): # Members in STARTUP2 or RECOVERING states are shown in the primary's # isMaster response, but aren't secondaries and shouldn't be read from. # Verify that if a secondary goes into RECOVERING mode, the Monitor removes # it from the set of readers. def setUp(self): members = [{}, {'priority': 0}, {'priority': 0}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_monitor_removes_recovering_member(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = [_partition_node(s) for s in secondaries] utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) sleep(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, _partition_node(secondary), mode) def tearDown(self): self.c.close() ha_tools.kill_all_members()
def getMongoDB(): conn = ReplicaSetConnection( "192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs') conn.read_preference = ReadPreference.SECONDARY_PREFERRED db = conn['bugreporter'] collection = db['original_data'] return collection
class TestReplicaSetRequest(unittest.TestCase): def setUp(self): members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) def test_request_during_failover(self): primary = _partition_node(ha_tools.get_primary()) secondary = _partition_node(ha_tools.get_random_secondary()) self.assertTrue(self.c.auto_start_request) self.assertTrue(self.c.in_request()) primary_pool = self.c._MongoReplicaSetClient__members[primary].pool secondary_pool = self.c._MongoReplicaSetClient__members[secondary].pool # Trigger start_request on primary pool utils.assertReadFrom(self, self.c, primary, PRIMARY) self.assertTrue(primary_pool.in_request()) # Fail over ha_tools.kill_primary() patience_seconds = 60 for _ in range(patience_seconds): sleep(1) try: if ha_tools.ha_tools_debug: print 'Waiting for failover' if ha_tools.get_primary(): # We have a new primary break except ConnectionFailure: pass else: self.fail("Problem with test: No new primary after %s seconds" % patience_seconds) try: # Trigger start_request on secondary_pool, which is becoming new # primary self.c.test.test.find_one() except AutoReconnect: # We've noticed the failover now pass # The old secondary is now primary utils.assertReadFrom(self, self.c, secondary, PRIMARY) self.assertTrue(self.c.in_request()) self.assertTrue(secondary_pool.in_request()) def tearDown(self): self.c.close() ha_tools.kill_all_members()
class TestPassiveAndHidden(unittest.TestCase): def setUp(self): members = [{}, { 'priority': 0 }, { 'arbiterOnly': True }, { 'priority': 0, 'hidden': True }, { 'priority': 0, 'slaveDelay': 5 }] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = self.c.pymongo_test w = len(self.c.secondaries) + 1 db.test.remove({}, safe=True, w=w) db.test.insert({'foo': 'bar'}, safe=True, w=w) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] hidden = ha_tools.get_hidden_members() hidden = [_partition_node(member) for member in hidden] self.assertEqual(self.c.secondaries, set(passives)) for mode in (ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED): db.read_preference = mode for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in passives) self.assertTrue(cursor._Cursor__connection_id not in hidden) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) db.read_preference = ReadPreference.SECONDARY_PREFERRED for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, self.c.primary) def tearDown(self): self.c.close() ha_tools.kill_all_members()
class TestReplicaSetAuth(unittest.TestCase): def setUp(self): members = [ {}, { 'priority': 0 }, { 'priority': 0 }, ] res = ha_tools.start_replica_set(members, auth=True) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth try: self.c.admin.add_user('admin', 'adminpass') except: # SERVER-4225 pass self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout() def test_auth_during_failover(self): self.assertTrue(self.db.authenticate('user', 'userpass')) self.assertTrue( self.db.foo.insert({'foo': 'bar'}, safe=True, w=3, wtimeout=1000)) self.db.logout() self.assertRaises(OperationFailure, self.db.foo.find_one) primary = '%s:%d' % self.c.primary ha_tools.kill_members([primary], 2) # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # Make sure we can still authenticate self.assertTrue(self.db.authenticate('user', 'userpass')) # And still query. self.db.read_preference = ReadPreference.PRIMARY_PREFERRED self.assertEqual('bar', self.db.foo.find_one()['foo']) def tearDown(self): self.c.close() ha_tools.kill_all_members()
def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times()
class TestPassiveAndHidden(unittest.TestCase): def setUp(self): members = [{}, {'priority': 0}, {'arbiterOnly': True}, {'priority': 0, 'hidden': True}, {'priority': 0, 'slaveDelay': 5} ] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_passive_and_hidden(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = self.c.pymongo_test w = len(self.c.secondaries) + 1 db.test.remove({}, safe=True, w=w) db.test.insert({'foo': 'bar'}, safe=True, w=w) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] hidden = ha_tools.get_hidden_members() hidden = [_partition_node(member) for member in hidden] self.assertEqual(self.c.secondaries, set(passives)) for mode in ( ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED ): db.read_preference = mode for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in passives) self.assertTrue(cursor._Cursor__connection_id not in hidden) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) db.read_preference = ReadPreference.SECONDARY_PREFERRED for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, self.c.primary) def tearDown(self): self.c.close() ha_tools.kill_all_members()
def test_secondary_failure(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries def readers_changed(): for _ in xrange(20): if c.secondaries != secondaries: return True sleep(1) return False killed = replset_tools.kill_secondary() sleep(5) # Let monitor thread notice change self.assertTrue(bool(len(killed))) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed()) secondaries = c.secondaries replset_tools.restart_members(killed) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed())
def test_secondary_connection(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Force replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST).is_primary) self.assertTrue(Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY).is_primary) for kwargs in [ {'read_preference': ReadPreference.PRIMARY_PREFERRED}, {'read_preference': ReadPreference.SECONDARY}, {'read_preference': ReadPreference.SECONDARY_PREFERRED}, {'read_preference': ReadPreference.NEAREST}, {'slave_okay': True}, ]: conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter secondary_host = ha_tools.get_arbiters()[0] host, port = ha_tools.get_arbiters()[0].split(':') port = int(port) conn = Connection(host, port) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port)
def test_passive_and_hidden(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name) db = c.pymongo_test db.test.remove({}, safe=True, w=len(c.secondaries)) w = len(c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) db.read_preference = ReadPreference.SECONDARY passives = replset_tools.get_passives() passives = [_partition_node(member) for member in passives] hidden = replset_tools.get_hidden_members() hidden = [_partition_node(member) for member in hidden] self.assertEqual(c.secondaries, set(passives)) for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id not in hidden) replset_tools.kill_members(replset_tools.get_passives(), 2) for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, c.primary)
def test_secondary_failure(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries def readers_changed(): for _ in xrange(20): if c.secondaries != secondaries: return True sleep(1) return False killed = ha_tools.kill_secondary() sleep(2 * MONITOR_INTERVAL) self.assertTrue(bool(len(killed))) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed()) secondaries = c.secondaries ha_tools.restart_members([killed]) self.assertEqual(primary, c.primary) self.assertTrue(readers_changed())
def test_passive_and_hidden(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) db = self.c.pymongo_test w = len(self.c.secondaries) + 1 db.test.remove({}, safe=True, w=w) db.test.insert({'foo': 'bar'}, safe=True, w=w) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] hidden = ha_tools.get_hidden_members() hidden = [_partition_node(member) for member in hidden] self.assertEqual(self.c.secondaries, set(passives)) for mode in ( ReadPreference.SECONDARY, ReadPreference.SECONDARY_PREFERRED ): db.read_preference = mode for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in passives) self.assertTrue(cursor._Cursor__connection_id not in hidden) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) db.read_preference = ReadPreference.SECONDARY_PREFERRED for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, self.c.primary)
def test_writes_with_failover(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) primary = c.primary db = c.pymongo_test w = len(c.secondaries) + 1 db.test.remove({}, safe=True, w=w) db.test.insert({'foo': 'bar'}, safe=True, w=w) self.assertEqual('bar', db.test.find_one()['foo']) def try_write(): for _ in xrange(30): try: db.test.insert({'bar': 'baz'}, safe=True) return True except AutoReconnect: sleep(1) return False killed = ha_tools.kill_primary(9) self.assertTrue(bool(len(killed))) self.assertTrue(try_write()) self.assertTrue(primary != c.primary) self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
def test_read_with_failover(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) self.assertTrue(bool(len(c.secondaries))) def iter_cursor(cursor): for _ in cursor: pass return True db = c.pymongo_test w = len(c.secondaries) + 1 db.test.remove({}, safe=True, w=w) # Force replication db.test.insert([{'foo': i} for i in xrange(10)], safe=True, w=w) self.assertEqual(10, db.test.count()) db.read_preference = ReadPreference.SECONDARY_PREFERRED cursor = db.test.find().batch_size(5) cursor.next() self.assertEqual(5, cursor._Cursor__retrieved) self.assertTrue(cursor._Cursor__connection_id in c.secondaries) ha_tools.kill_primary() # Primary failure shouldn't interrupt the cursor self.assertTrue(iter_cursor(cursor)) self.assertEqual(10, cursor._Cursor__retrieved)
class TestReplicaSetAuth(unittest.TestCase): def setUp(self): members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth try: self.c.admin.add_user('admin', 'adminpass') except: # SERVER-4225 pass self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout() def test_auth_during_failover(self): self.assertTrue(self.db.authenticate('user', 'userpass')) self.assertTrue(self.db.foo.insert({'foo': 'bar'}, safe=True, w=3, wtimeout=1000)) self.db.logout() self.assertRaises(OperationFailure, self.db.foo.find_one) primary = '%s:%d' % self.c.primary ha_tools.kill_members([primary], 2) # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # Make sure we can still authenticate self.assertTrue(self.db.authenticate('user', 'userpass')) # And still query. self.db.read_preference = ReadPreference.PRIMARY_PREFERRED self.assertEqual('bar', self.db.foo.find_one()['foo']) def tearDown(self): self.c.close() ha_tools.kill_all_members()
def test_alive(self): primary = ha_tools.get_primary() secondary = ha_tools.get_random_secondary() primary_cx = Connection(primary, use_greenlets=use_greenlets) secondary_cx = Connection(secondary, use_greenlets=use_greenlets) rsc = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) try: self.assertTrue(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertTrue(rsc.alive()) ha_tools.kill_primary() time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertTrue(secondary_cx.alive()) self.assertFalse(rsc.alive()) ha_tools.kill_members([secondary], 2) time.sleep(0.5) self.assertFalse(primary_cx.alive()) self.assertFalse(secondary_cx.alive()) self.assertFalse(rsc.alive()) finally: rsc.close()
def setUp(self): members = [{}, {"priority": 0}, {"priority": 0}] res = ha_tools.start_replica_set(members, auth=True) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth self.c.admin.add_user("admin", "adminpass") self.c.admin.authenticate("admin", "adminpass") self.db = self.c.pymongo_ha_auth self.db.add_user("user", "userpass") self.c.admin.logout()
def test_passive_and_hidden(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) passives = ha_tools.get_passives() passives = [_partition_node(member) for member in passives] self.assertEqual(self.c.secondaries, set(passives)) for mode in SECONDARY, SECONDARY_PREFERRED: utils.assertReadFromAll(self, self.c, passives, mode) ha_tools.kill_members(ha_tools.get_passives(), 2) sleep(2 * MONITOR_INTERVAL) utils.assertReadFrom(self, self.c, self.c.primary, SECONDARY_PREFERRED)
def getConn(self): ''' 获取 primary 的连接信息 ''' if self._repl: client = ReplicaSetConnection( self._addr, replicaSet=self._repl, read_preference=pymongo.read_preferences.ReadPreference.PRIMARY ) else: client = MongoClient(self._addr) if self._user and self._pwd: client.admin.authenticate(self._user, self._pwd) return client
def test_primary_stepdown(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries def primary_changed(): for _ in xrange(30): if c.primary != primary: return True time.sleep(1) return False replset_tools.stepdown_primary() self.assertTrue(primary_changed()) self.assertTrue(secondaries != c.secondaries)
def init(self): logger.info( "[Trending broker] I init the %s server connection to %s:%s (%s)" % (self.get_name(), self.backend, self.uri, self.replica_set)) if self.replica_set: self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, safe=True) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.con = Connection(self.uri, safe=True) else: self.con = Connection(self.uri, safe=True) # Open a connection self.db = getattr(self.con, self.database) self.col = self.db['trending']
def __init__(self, conn_str, db_name): try: if conn_str.find("replicaSet") == -1: _conn = Connection( conn_str, max_pool_size=30, safe=True, read_preference=ReadPreference.SECONDARY_ONLY) else: _conn = ReplicaSetConnection( conn_str, max_pool_size=30, safe=True, read_preference=ReadPreference.SECONDARY_ONLY) self._db = _conn[db_name] except Exception, e: logger.exception('Can not connect to mongodb: %s' % e) raise e
def test_monitor_removes_recovering_member(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) secondaries = ha_tools.get_secondaries() for mode in SECONDARY, SECONDARY_PREFERRED: partitioned_secondaries = [_partition_node(s) for s in secondaries] utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode) secondary, recovering_secondary = secondaries ha_tools.set_maintenance(recovering_secondary, True) sleep(2 * MONITOR_INTERVAL) for mode in SECONDARY, SECONDARY_PREFERRED: # Don't read from recovering member utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
def test_primary_failure(self): c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries # Wait for new primary to be elected def primary_changed(): for _ in xrange(30): if c.primary and c.primary != primary: return True sleep(1) return False killed = ha_tools.kill_primary() self.assertTrue(bool(len(killed))) self.assertTrue(primary_changed()) self.assertNotEqual(secondaries, c.secondaries)
def test_simple_ops(self): if not have_ssl: raise SkipTest() try: conn = Connection(connectTimeoutMS=100, ssl=True) # MongoDB not configured for SSL? except ConnectionFailure: raise SkipTest() response = conn.admin.command('ismaster') if 'setName' in response: conn = ReplicaSetConnection(replicaSet=response['setName'], w=len(response['hosts']), ssl=True) db = conn.pymongo_ssl_test self.assert_(db.test.insert({'ssl': True}, safe=True)) self.assertTrue(db.test.find_one()['ssl']) conn.drop_database(db)
def setUp(self): members = [ # primary {"tags": {"dc": "ny", "name": "primary"}}, # secondary {"tags": {"dc": "la", "name": "secondary"}, "priority": 0}, # other_secondary {"tags": {"dc": "ny", "name": "other_secondary"}, "priority": 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual("primary", self.primary_tags["name"]) self.primary_dc = {"dc": self.primary_tags["dc"]} secondaries = ha_tools.get_secondaries() (secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "secondary"] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {"dc": self.secondary_tags["dc"]} (other_secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "other_secondary"] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {"dc": self.other_secondary_tags["dc"]} self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert([{"foo": i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times()
def test_primary_failure(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries def primary_changed(): for _ in xrange(30): if c.primary != primary: return True sleep(1) return False killed = replset_tools.kill_primary() sleep(5) # Let monitor thread notice change self.assertTrue(bool(len(killed))) self.assertTrue(primary_changed()) self.assertTrue(secondaries != c.secondaries)
def init(self): """ Called by Scheduler to say 'let's prepare yourself guy' """ logger.debug("Initialization of the mongodb module") if self.replica_set: self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, fsync=True) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.con = Connection(self.uri, fsync=True) else: self.con = Connection(self.uri) #self.con = Connection(self.uri) # Open a gridfs connection self.db = getattr(self.con, self.database) self.hosts_fs = GridFS(self.db, collection='retention_hosts') self.services_fs = GridFS(self.db, collection='retention_services')
def test_primary_stepdown(self): c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(c.secondaries))) primary = c.primary secondaries = c.secondaries.copy() def primary_changed(): for _ in xrange(30): if c.primary != primary: return True sleep(1) return False ha_tools.stepdown_primary() self.assertTrue(primary_changed()) # There can be a delay between finding the primary and updating # secondaries sleep(5) self.assertNotEqual(secondaries, c.secondaries)
def init(self): """ Called by broker to say 'let's prepare yourself guy' """ logger.debug("Initialization of the mongodb snapshot module") if self.replica_set: self.con = ReplicaSetConnection(self.uri, replicaSet=self.replica_set, fsync=False) else: # Old versions of pymongo do not known about fsync if ReplicaSetConnection: self.con = Connection(self.uri, fsync=False) else: self.con = Connection(self.uri) self.db = getattr(self.con, self.database) self.snapshots = getattr(self.db, 'snapshots') self.snapshots.ensure_index([('host_name', pymongo.DESCENDING), ('snapshot_time', pymongo.DESCENDING)])
def getConnection(self, dbName, isSECONDARYONLY=True): ''' @summary: 获取连接 @param dbName: 要连接的数据库 ''' tryMaxNum = 100 tryNum = 0 while tryNum < tryMaxNum: tryNum = tryNum + 1 try: conn = ReplicaSetConnection(self.mongoDbHost, replicaSet=self.replicaSetName) self.DB = conn[dbName] if isSECONDARYONLY: self.DB.read_preference = ReadPreference.PRIMARY else: self.DB.read_preference = ReadPreference.PRIMARY self.DB.authenticate(self.mongoDbUser, self.mongoDbPassword) break except Exception, e: time.sleep(5)
def setUp(self): members = [ {}, {'priority': 0}, {'priority': 0}, ] res = ha_tools.start_replica_set(members, auth=True) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets) # Add an admin user to enable auth try: self.c.admin.add_user('admin', 'adminpass') except: # SERVER-4225 pass self.c.admin.authenticate('admin', 'adminpass') self.db = self.c.pymongo_ha_auth self.db.add_user('user', 'userpass') self.c.admin.logout()
def getMongoDB(): conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs') conn.read_preference = ReadPreference.SECONDARY_PREFERRED db = conn['bugreporter'] collection = db['original_data'] return collection
def test_secondary_connection(self): self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Force replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets).is_primary) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST).is_primary) self.assertTrue( Connection(primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY).is_primary) for kwargs in [ { 'read_preference': ReadPreference.PRIMARY_PREFERRED }, { 'read_preference': ReadPreference.SECONDARY }, { 'read_preference': ReadPreference.SECONDARY_PREFERRED }, { 'read_preference': ReadPreference.NEAREST }, { 'slave_okay': True }, ]: conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter host, port = ha_tools.get_arbiters()[0].split(':') port = int(port) conn = Connection(host, port) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port)
class TestReadPreference(unittest.TestCase): def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): Member._host_to_ping_time[host] = ping_time_seconds def clear_ping_times(self): Member._host_to_ping_time.clear() def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy modes and hosts into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() def tearDown(self): self.c.close() ha_tools.kill_all_members() self.clear_ping_times()
def getDB(self): #return Connection('192.168.7.201',27017)['bugreporter'] #return Connection('localhost',27017)['bugreporter'] conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs') conn.read_preference = ReadPreference.SECONDARY_PREFERRED return conn['bugreporter']
class TestReadPreference(unittest.TestCase): def setUp(self): members = [ # primary {'tags': {'dc': 'ny', 'name': 'primary'}}, # secondary {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0}, # other_secondary {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0}, ] res = ha_tools.start_replica_set(members) self.seed, self.name = res primary = ha_tools.get_primary() self.primary = _partition_node(primary) self.primary_tags = ha_tools.get_tags(primary) # Make sure priority worked self.assertEqual('primary', self.primary_tags['name']) self.primary_dc = {'dc': self.primary_tags['dc']} secondaries = ha_tools.get_secondaries() (secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'secondary'] self.secondary = _partition_node(secondary) self.secondary_tags = ha_tools.get_tags(secondary) self.secondary_dc = {'dc': self.secondary_tags['dc']} (other_secondary, ) = [ s for s in secondaries if ha_tools.get_tags(s)['name'] == 'other_secondary'] self.other_secondary = _partition_node(other_secondary) self.other_secondary_tags = ha_tools.get_tags(other_secondary) self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']} self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.db = self.c.pymongo_test self.w = len(self.c.secondaries) + 1 self.db.test.remove({}, safe=True, w=self.w) self.db.test.insert( [{'foo': i} for i in xrange(10)], safe=True, w=self.w) self.clear_ping_times() def set_ping_time(self, host, ping_time_seconds): Member._host_to_ping_time[host] = ping_time_seconds def clear_ping_times(self): Member._host_to_ping_time.clear() def test_read_preference(self): # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy modes and hosts into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, {'dc': 'ny'}]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll( [primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000*1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom( other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue(Connection( unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True ).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times() def tearDown(self): self.c.close() ha_tools.kill_all_members() self.clear_ping_times()
def getConn(self): #return Connection('192.168.7.201',27017) conn=ReplicaSetConnection("192.168.5.60:27017,192.168.7.52:27017,192.168.5.156:27017", replicaSet='ats_rs') conn.read_preference = ReadPreference.SECONDARY_PREFERRED return conn
def test_secondary_connection(self): self.c = ReplicaSetConnection( self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Wait for replication... w = len(self.c.secondaries) + 1 db.test.insert({'foo': 'bar'}, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(':') primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(':') secondary_port = int(secondary_port) arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':') arbiter_port = int(arbiter_port) # A Connection succeeds no matter the read preference for kwargs in [ {'read_preference': ReadPreference.PRIMARY}, {'read_preference': ReadPreference.PRIMARY_PREFERRED}, {'read_preference': ReadPreference.SECONDARY}, {'read_preference': ReadPreference.SECONDARY_PREFERRED}, {'read_preference': ReadPreference.NEAREST}, {'slave_okay': True} ]: conn = Connection(primary_host, primary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(primary_host, conn.host) self.assertEqual(primary_port, conn.port) self.assertTrue(conn.is_primary) # Direct connection to primary can be queried with any read pref self.assertTrue(conn.pymongo_test.test.find_one()) conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) # Direct connection to secondary can be queried with any read pref # but PRIMARY if kwargs.get('read_preference') != ReadPreference.PRIMARY: self.assertTrue(conn.pymongo_test.test.find_one()) else: self.assertRaises( AutoReconnect, conn.pymongo_test.test.find_one) # Since an attempt at an acknowledged write to a secondary from a # direct connection raises AutoReconnect('not master'), Connection # should do the same for unacknowledged writes. try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.message) else: self.fail( 'Unacknowledged insert into secondary connection %s should' 'have raised exception' % conn) # Test direct connection to an arbiter conn = Connection(arbiter_host, arbiter_port, **kwargs) self.assertEqual(arbiter_host, conn.host) self.assertEqual(arbiter_port, conn.port) self.assertFalse(conn.is_primary) # See explanation above try: conn.pymongo_test.test.insert({}, safe=False) except AutoReconnect, e: self.assertEqual('not master', e.message)
def test_read_preference(self): c = ReplicaSetConnection(self.seed, replicaSet=self.name) self.assertTrue(bool(len(c.secondaries))) db = c.pymongo_test db.test.remove({}, safe=True, w=len(c.secondaries)) # Force replication... w = len(c.secondaries) + 1 db.test.insert({'foo': 'bar'}, safe=True, w=w) # Test direct connection to a secondary host, port = replset_tools.get_secondaries()[0].split(':') port = int(port) conn = Connection(host, port, slave_okay=True) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port) self.assert_(conn.pymongo_test.test.find_one()) conn = Connection(host, port, read_preference=ReadPreference.SECONDARY) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter host = replset_tools.get_arbiters()[0] self.assertRaises(ConnectionFailure, Connection, host) # Test PRIMARY for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, c.primary) # Test SECONDARY with a secondary db.read_preference = ReadPreference.SECONDARY for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in c.secondaries) # Test SECONDARY_ONLY with a secondary db.read_preference = ReadPreference.SECONDARY_ONLY for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertTrue(cursor._Cursor__connection_id in c.secondaries) # Test SECONDARY with no secondary killed = replset_tools.kill_all_secondaries() self.assertTrue(bool(len(killed))) db.read_preference = ReadPreference.SECONDARY for _ in xrange(10): cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, c.primary) # Test SECONDARY_ONLY with no secondary db.read_preference = ReadPreference.SECONDARY_ONLY for _ in xrange(10): cursor = db.test.find() self.assertRaises(AutoReconnect, cursor.next) replset_tools.restart_members(killed) # Test PRIMARY with no primary (should raise an exception) db.read_preference = ReadPreference.PRIMARY cursor = db.test.find() cursor.next() self.assertEqual(cursor._Cursor__connection_id, c.primary) killed = replset_tools.kill_primary() self.assertTrue(bool(len(killed))) self.assertRaises(AutoReconnect, db.test.find_one)
def test_read_preference(self): # This is long, but we put all the tests in one function to save time # on setUp, which takes about 30 seconds to bring up a replica set. # We pass through four states: # # 1. A primary and two secondaries # 2. Primary down # 3. Primary up, one secondary down # 4. Primary up, all secondaries down # # For each state, we verify the behavior of PRIMARY, # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets, auto_start_request=False) def assertReadFrom(member, *args, **kwargs): utils.assertReadFrom(self, c, member, *args, **kwargs) def assertReadFromAll(members, *args, **kwargs): utils.assertReadFromAll(self, c, members, *args, **kwargs) def unpartition_node(node): host, port = node return '%s:%s' % (host, port) # To make the code terser, copy modes and hosts into local scope PRIMARY = ReadPreference.PRIMARY PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED SECONDARY = ReadPreference.SECONDARY SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED NEAREST = ReadPreference.NEAREST primary = self.primary secondary = self.secondary other_secondary = self.other_secondary bad_tag = {'bad': 'tag'} # 1. THREE MEMBERS UP ------------------------------------------------- # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED # Trivial: mode and tags both match assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Secondary matches but not primary, choose primary assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc) # Chooses primary, ignoring tag sets assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag) assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}]) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Multiple tags assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags) # Fall back to primary if it's the only one matching the tags assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'}) # No matching secondaries assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag) # Fall back from non-matching tag set to matching set assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED, [bad_tag, {}]) assertReadFrom(other_secondary, SECONDARY_PREFERRED, [bad_tag, { 'dc': 'ny' }]) # NEAREST self.clear_ping_times() assertReadFromAll([primary, secondary, other_secondary], NEAREST) assertReadFromAll([primary, other_secondary], NEAREST, [bad_tag, { 'dc': 'ny' }]) self.set_ping_time(primary, 0) self.set_ping_time(secondary, .03) # 30 ms self.set_ping_time(other_secondary, 10) # Nearest member, no tags assertReadFrom(primary, NEAREST) # Tags override nearness assertReadFrom(primary, NEAREST, {'name': 'primary'}) assertReadFrom(secondary, NEAREST, self.secondary_dc) # Make secondary fast self.set_ping_time(primary, .03) # 30 ms self.set_ping_time(secondary, 0) assertReadFrom(secondary, NEAREST) # Other secondary fast self.set_ping_time(secondary, 10) self.set_ping_time(other_secondary, 0) assertReadFrom(other_secondary, NEAREST) # High secondaryAcceptableLatencyMS, should read from all members assertReadFromAll([primary, secondary, other_secondary], NEAREST, secondary_acceptable_latency_ms=1000 * 1000) self.clear_ping_times() assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}]) # 2. PRIMARY DOWN ----------------------------------------------------- killed = ha_tools.kill_primary() # Let monitor notice primary's gone sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(None, PRIMARY) # PRIMARY_PREFERRED # No primary, choose matching secondary assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED) assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'}) # No primary or matching secondary assertReadFrom(None, PRIMARY_PREFERRED, bad_tag) # SECONDARY assertReadFromAll([secondary, other_secondary], SECONDARY) # Only primary matches assertReadFrom(None, SECONDARY, {'name': 'primary'}) # No matching secondaries assertReadFrom(None, SECONDARY, bad_tag) # SECONDARY_PREFERRED assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED) # Mode and tags both match assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST self.clear_ping_times() assertReadFromAll([secondary, other_secondary], NEAREST) # 3. PRIMARY UP, ONE SECONDARY DOWN ----------------------------------- ha_tools.restart_members([killed]) for _ in range(30): if ha_tools.get_primary(): break sleep(1) else: self.fail("Primary didn't come back up") ha_tools.kill_members([unpartition_node(secondary)], 2) self.assertTrue( Connection(unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True).admin.command('ismaster')['ismaster']) sleep(2 * MONITOR_INTERVAL) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) # SECONDARY assertReadFrom(other_secondary, SECONDARY) assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc) # Only the down secondary matches assertReadFrom(None, SECONDARY, {'name': 'secondary'}) # SECONDARY_PREFERRED assertReadFrom(other_secondary, SECONDARY_PREFERRED) assertReadFrom(other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc) # The secondary matching the tag is down, use primary assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) # NEAREST assertReadFromAll([primary, other_secondary], NEAREST) assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'}) assertReadFrom(primary, NEAREST, {'name': 'primary'}) # 4. PRIMARY UP, ALL SECONDARIES DOWN --------------------------------- ha_tools.kill_members([unpartition_node(other_secondary)], 2) self.assertTrue( Connection(unpartition_node(primary), use_greenlets=use_greenlets, slave_okay=True).admin.command('ismaster')['ismaster']) # PRIMARY assertReadFrom(primary, PRIMARY) # PRIMARY_PREFERRED assertReadFrom(primary, PRIMARY_PREFERRED) assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc) # SECONDARY assertReadFrom(None, SECONDARY) assertReadFrom(None, SECONDARY, self.other_secondary_dc) assertReadFrom(None, SECONDARY, {'dc': 'ny'}) # SECONDARY_PREFERRED assertReadFrom(primary, SECONDARY_PREFERRED) assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc) assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'}) assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'}) # NEAREST assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) assertReadFrom(None, NEAREST, {'name': 'secondary'}) # Even if primary's slow, still read from it self.set_ping_time(primary, 100) assertReadFrom(primary, NEAREST) assertReadFrom(None, NEAREST, self.secondary_dc) self.clear_ping_times()
import urllib2 from bs4 import BeautifulSoup import time if __name__ == '__main__': # The Mongo database to connect to db_to_use = sys.argv[1] # The collection to use collection_to_use = sys.argv[2] # The LinkedIn URL field in your Mongo database linkedin_url_field = sys.argv[3] # Connect to the database # client = MongoClient() client = ReplicaSetConnection('localhost:27017', replicaSet='rep01') db = client[db_to_use] collection = db[collection_to_use] # Get a list of all the records in the database - we need to update everything # Fetch the record and get the linkedin url for item in collection.find(): # Fetch the record and get the linkedin url record_id = ObjectId(item["_id"]) url_to_get = item[linkedin_url_field] print "Current ObjectID: " + str(record_id) print "Current URL: " + url_to_get if url_to_get != "N/A":
def setUp(self): members = [{}, {}, {'arbiterOnly': True}] res = ha_tools.start_replica_set(members) self.c = ReplicaSetConnection(res[0], replicaSet=res[1], use_greenlets=use_greenlets)
class TestSecondaryConnection(unittest.TestCase): def setUp(self): members = [{}, {}, {"arbiterOnly": True}] res = ha_tools.start_replica_set(members) self.seed, self.name = res def test_secondary_connection(self): self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets) self.assertTrue(bool(len(self.c.secondaries))) db = self.c.pymongo_test db.test.remove({}, safe=True, w=len(self.c.secondaries)) # Force replication... w = len(self.c.secondaries) + 1 db.test.insert({"foo": "bar"}, safe=True, w=w) # Test direct connection to a primary or secondary primary_host, primary_port = ha_tools.get_primary().split(":") primary_port = int(primary_port) (secondary_host, secondary_port) = ha_tools.get_secondaries()[0].split(":") secondary_port = int(secondary_port) self.assertTrue(Connection(primary_host, primary_port, use_greenlets=use_greenlets).is_primary) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.PRIMARY_PREFERRED, ).is_primary ) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY_PREFERRED, ).is_primary ) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.NEAREST ).is_primary ) self.assertTrue( Connection( primary_host, primary_port, use_greenlets=use_greenlets, read_preference=ReadPreference.SECONDARY ).is_primary ) for kwargs in [ {"read_preference": ReadPreference.PRIMARY_PREFERRED}, {"read_preference": ReadPreference.SECONDARY}, {"read_preference": ReadPreference.SECONDARY_PREFERRED}, {"read_preference": ReadPreference.NEAREST}, {"slave_okay": True}, ]: conn = Connection(secondary_host, secondary_port, use_greenlets=use_greenlets, **kwargs) self.assertEqual(secondary_host, conn.host) self.assertEqual(secondary_port, conn.port) self.assertFalse(conn.is_primary) self.assert_(conn.pymongo_test.test.find_one()) # Test direct connection to an arbiter secondary_host = ha_tools.get_arbiters()[0] host, port = ha_tools.get_arbiters()[0].split(":") port = int(port) conn = Connection(host, port) self.assertEqual(host, conn.host) self.assertEqual(port, conn.port) def tearDown(self): self.c.close() ha_tools.kill_all_members()
f.close() @classmethod def loadGlobal(cls, filename): """ Loads a previously created global_storage file """ import json f = open(filename, 'r') cls.global_storage = json.load(f) f.close() if __name__ == "__main__": import json from pymongo import ReplicaSetConnection, ReadPreference connection = ReplicaSetConnection("clutch:27017", replicaSet="geo") connection.read_preference = ReadPreference.SECONDARY f = open("words.json",'r') word_dict = json.loads(f.read()) f.close() #GEOvector.loadGlobal("gv_test.json" ) gv = GEOvector('GSE2395',word_dict, mongo_conn=connection, cache=False) print gv.getSparse() print "1" print sum(gv) print sum(gv)