def test_read_with_failover(self):
        c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
            auto_start_request=False)
        self.assertTrue(bool(len(c.secondaries)))

        def iter_cursor(cursor):
            for doc in cursor:
                pass
            return True

        db = c.pymongo_test
        w = len(c.secondaries) + 1
        db.test.remove({}, safe=True, w=w)
        # Force replication
        db.test.insert([{'foo': i} for i in xrange(10)], safe=True, w=w)
        self.assertEqual(10, db.test.count())

        db.read_preference = ReadPreference.SECONDARY_PREFERRED
        cursor = db.test.find().batch_size(5)
        cursor.next()
        self.assertEqual(5, cursor._Cursor__retrieved)
        self.assertTrue(cursor._Cursor__connection_id in c.secondaries)
        replset_tools.kill_primary()
        # Primary failure shouldn't interrupt the cursor
        self.assertTrue(iter_cursor(cursor))
        self.assertEqual(10, cursor._Cursor__retrieved)
Exemplo n.º 2
0
    def test_read_with_failover(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        self.assertTrue(bool(len(c.secondaries)))

        def iter_cursor(cursor):
            for doc in cursor:
                pass
            return True

        db = c.pymongo_test
        w = len(c.secondaries) + 1
        db.test.remove({}, safe=True, w=w)
        # Force replication
        db.test.insert([{'foo': i} for i in xrange(10)],
                       safe=True, w=w)
        self.assertEqual(10, db.test.count())

        db.read_preference = ReadPreference.SECONDARY
        cursor = db.test.find().batch_size(5)
        cursor.next()
        self.assertEqual(5, cursor._Cursor__retrieved)
        killed = replset_tools.kill_primary()
        # Primary failure shouldn't interrupt the cursor
        self.assertTrue(iter_cursor(cursor))
        self.assertEqual(10, cursor._Cursor__retrieved)
Exemplo n.º 3
0
    def test_writes_with_failover(self):
        c = ReplicaSetConnection(self.seed,
                                 replicaSet=self.name,
                                 use_greenlets=use_greenlets)
        primary = c.primary
        db = c.pymongo_test
        w = len(c.secondaries) + 1
        db.test.remove({}, safe=True, w=w)
        db.test.insert({'foo': 'bar'}, safe=True, w=w)
        self.assertEqual('bar', db.test.find_one()['foo'])

        def try_write():
            for _ in xrange(30):
                try:
                    db.test.insert({'bar': 'baz'}, safe=True)
                    return True
                except AutoReconnect:
                    sleep(1)
            return False

        killed = replset_tools.kill_primary(9)
        self.assertTrue(bool(len(killed)))
        self.assertTrue(try_write())
        self.assertTrue(primary != c.primary)
        self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
Exemplo n.º 4
0
    def test_primary_failure(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        secondaries = c.secondaries

        def primary_changed():
            for _ in xrange(20):
                if c.primary != primary:
                    return True
                time.sleep(1)
            return False

        killed = replset_tools.kill_primary()
        self.assertTrue(bool(len(killed)))
        self.assertTrue(primary_changed())
        self.assertTrue(secondaries != c.secondaries)
    def test_primary_failure(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        secondaries = c.secondaries

        def primary_changed():
            for _ in xrange(30):
                if c.primary != primary:
                    return True
                time.sleep(1)
            return False

        killed = replset_tools.kill_primary()
        self.assertTrue(bool(len(killed)))
        self.assertTrue(primary_changed())
        self.assertTrue(secondaries != c.secondaries)
    def test_primary_failure(self):
        c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        secondaries = c.secondaries

        # Wait for new primary to be elected
        def primary_changed():
            for _ in xrange(30):
                if c.primary and c.primary != primary:
                    return True
                sleep(1)
            return False

        killed = replset_tools.kill_primary()
        self.assertTrue(bool(len(killed)))
        self.assertTrue(primary_changed())
        self.assertNotEqual(secondaries, c.secondaries)
    def test_primary_failure(self):
        c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        secondaries = c.secondaries

        def primary_changed():
            for _ in xrange(30):
                if c.primary != primary:
                    return True
                sleep(1)
            return False

        killed = replset_tools.kill_primary()
        sleep(5) # Let monitor thread notice change
        self.assertTrue(bool(len(killed)))
        self.assertTrue(primary_changed())
        self.assertTrue(secondaries != c.secondaries)
Exemplo n.º 8
0
    def test_primary_failure(self):
        c = ReplicaSetConnection(self.seed,
                                 replicaSet=self.name,
                                 use_greenlets=use_greenlets)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        secondaries = c.secondaries

        def primary_changed():
            for _ in xrange(30):
                if c.primary != primary:
                    return True
                sleep(1)
            return False

        killed = replset_tools.kill_primary()
        sleep(5)  # Let monitor thread notice change
        self.assertTrue(bool(len(killed)))
        self.assertTrue(primary_changed())
        self.assertTrue(secondaries != c.secondaries)
    def test_writes_with_failover(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        primary = c.primary
        db = c.pymongo_test
        w = len(c.secondaries) + 1
        db.test.remove({}, safe=True, w=w)
        db.test.insert({'foo': 'bar'}, safe=True, w=w)
        self.assertEqual('bar', db.test.find_one()['foo'])

        def try_write():
            for _ in xrange(30):
                try:
                    db.test.insert({'bar': 'baz'}, safe=True)
                    return True
                except AutoReconnect:
                    time.sleep(1)
            return False

        killed = replset_tools.kill_primary(9)
        self.assertTrue(bool(len(killed)))
        self.assertTrue(try_write())
        self.assertTrue(primary != c.primary)
        self.assertEqual('baz', db.test.find_one({'bar': 'baz'})['bar'])
Exemplo n.º 10
0
    def test_read_preference(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        self.assertTrue(bool(len(c.secondaries)))
        db = c.pymongo_test
        db.test.remove({}, safe=True, w=len(c.secondaries))

        # Force replication...
        w = len(c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, safe=True, w=w)

        # Test direct connection to a secondary
        host, port = replset_tools.get_secondaries()[0].split(':')
        port = int(port)
        conn = Connection(host, port, slave_okay=True)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
        self.assert_(conn.pymongo_test.test.find_one())
        conn = Connection(host, port, read_preference=ReadPreference.SECONDARY)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
        self.assert_(conn.pymongo_test.test.find_one())

        # Test direct connection to an arbiter
        host = replset_tools.get_arbiters()[0]
        self.assertRaises(ConnectionFailure, Connection, host)

        # Test PRIMARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertEqual(cursor._Cursor__connection_id, c.primary)

        # Test SECONDARY with a secondary
        db.read_preference = ReadPreference.SECONDARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertTrue(cursor._Cursor__connection_id in c.secondaries)

        # Test SECONDARY_ONLY with a secondary
        db.read_preference = ReadPreference.SECONDARY_ONLY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertTrue(cursor._Cursor__connection_id in c.secondaries)

        # Test SECONDARY with no secondary
        killed = replset_tools.kill_all_secondaries()
        self.assertTrue(bool(len(killed)))
        db.read_preference = ReadPreference.SECONDARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertEqual(cursor._Cursor__connection_id, c.primary)

        # Test SECONDARY_ONLY with no secondary
        db.read_preference = ReadPreference.SECONDARY_ONLY
        for _ in xrange(10):
            cursor = db.test.find()
            self.assertRaises(AutoReconnect, cursor.next)

        replset_tools.restart_members(killed)
        # Test PRIMARY with no primary (should raise an exception)
        db.read_preference = ReadPreference.PRIMARY
        cursor = db.test.find()
        cursor.next()
        self.assertEqual(cursor._Cursor__connection_id, c.primary)
        killed = replset_tools.kill_primary()
        self.assertTrue(bool(len(killed)))
        self.assertRaises(AutoReconnect, db.test.find_one)
    def test_read_preference(self):
        # This is long, but we put all the tests in one function to save time
        # on setUp, which takes about 30 seconds to bring up a replica set.
        # We pass through four states:
        #
        #       1. A primary and two secondaries
        #       2. Primary down
        #       3. Primary up, one secondary down
        #       4. Primary up, all secondaries down
        #
        # For each state, we verify the behavior of PRIMARY,
        # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
        c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
            auto_start_request=False)

        def assertReadFrom(member, *args, **kwargs):
            utils.assertReadFrom(self, c, member, *args, **kwargs)

        def assertReadFromAll(members, *args, **kwargs):
            utils.assertReadFromAll(self, c, members, *args, **kwargs)

        def unpartition_node(node):
            host, port = node
            return '%s:%s' % (host, port)

        # To make the code terser, copy modes and hosts into local scope
        PRIMARY = ReadPreference.PRIMARY
        PRIMARY_PREFERRED = ReadPreference.PRIMARY_PREFERRED
        SECONDARY = ReadPreference.SECONDARY
        SECONDARY_PREFERRED = ReadPreference.SECONDARY_PREFERRED
        NEAREST = ReadPreference.NEAREST
        
        primary = self.primary
        secondary = self.secondary
        other_secondary = self.other_secondary

        bad_tag = {'bad': 'tag'}
                
        # 1. THREE MEMBERS UP -------------------------------------------------
        #       PRIMARY
        assertReadFrom(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        # Trivial: mode and tags both match
        assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Secondary matches but not primary, choose primary
        assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)

        # Chooses primary, ignoring tag sets
        assertReadFrom(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Chooses primary, ignoring tag sets
        assertReadFrom(primary, PRIMARY_PREFERRED, bad_tag)
        assertReadFrom(primary, PRIMARY_PREFERRED, [bad_tag, {}])

        #       SECONDARY
        assertReadFromAll([secondary, other_secondary], SECONDARY)

        #       SECONDARY_PREFERRED
        assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)

        # Multiple tags
        assertReadFrom(secondary, SECONDARY_PREFERRED, self.secondary_tags)

        # Fall back to primary if it's the only one matching the tags
        assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'primary'})

        # No matching secondaries
        assertReadFrom(primary, SECONDARY_PREFERRED, bad_tag)

        # Fall back from non-matching tag set to matching set
        assertReadFromAll([secondary, other_secondary],
            SECONDARY_PREFERRED, [bad_tag, {}])

        assertReadFrom(other_secondary,
            SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])

        #       NEAREST
        self.clear_ping_times()

        assertReadFromAll([primary, secondary, other_secondary], NEAREST)

        assertReadFromAll([primary, other_secondary],
            NEAREST, [bad_tag, {'dc': 'ny'}])

        self.set_ping_time(primary, 0)
        self.set_ping_time(secondary, .03) # 30 ms
        self.set_ping_time(other_secondary, 10)

        # Nearest member, no tags
        assertReadFrom(primary, NEAREST)

        # Tags override nearness
        assertReadFrom(primary, NEAREST, {'name': 'primary'})
        assertReadFrom(secondary, NEAREST, self.secondary_dc)

        # Make secondary fast
        self.set_ping_time(primary, .03) # 30 ms
        self.set_ping_time(secondary, 0)

        assertReadFrom(secondary, NEAREST)

        # Other secondary fast
        self.set_ping_time(secondary, 10)
        self.set_ping_time(other_secondary, 0)

        assertReadFrom(other_secondary, NEAREST)

        # High secondaryAcceptableLatencyMS, should read from all members
        assertReadFromAll(
            [primary, secondary, other_secondary],
            NEAREST, secondary_acceptable_latency_ms=1000*1000)

        self.clear_ping_times()

        assertReadFromAll([primary, other_secondary], NEAREST, [{'dc': 'ny'}])

        # 2. PRIMARY DOWN -----------------------------------------------------
        killed = replset_tools.kill_primary()

        # Let monitor notice primary's gone
        sleep(2 * MONITOR_INTERVAL)

        #       PRIMARY
        assertReadFrom(None, PRIMARY)

        #       PRIMARY_PREFERRED
        # No primary, choose matching secondary
        assertReadFromAll([secondary, other_secondary], PRIMARY_PREFERRED)
        assertReadFrom(secondary, PRIMARY_PREFERRED, {'name': 'secondary'})

        # No primary or matching secondary
        assertReadFrom(None, PRIMARY_PREFERRED, bad_tag)

        #       SECONDARY
        assertReadFromAll([secondary, other_secondary], SECONDARY)

        # Only primary matches
        assertReadFrom(None, SECONDARY, {'name': 'primary'})

        # No matching secondaries
        assertReadFrom(None, SECONDARY, bad_tag)

        #       SECONDARY_PREFERRED
        assertReadFromAll([secondary, other_secondary], SECONDARY_PREFERRED)

        # Mode and tags both match
        assertReadFrom(secondary, SECONDARY_PREFERRED, {'name': 'secondary'})

        #       NEAREST
        self.clear_ping_times()

        assertReadFromAll([secondary, other_secondary], NEAREST)
        
        # 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
        replset_tools.restart_members([killed])

        for _ in range(30):
            if replset_tools.get_primary():
                break
            sleep(1)
        else:
            self.fail("Primary didn't come back up")

        replset_tools.kill_members([unpartition_node(secondary)], 2)
        self.assertTrue(Connection(
            unpartition_node(primary), use_greenlets=use_greenlets,
            slave_okay=True
        ).admin.command('ismaster')['ismaster'])

        sleep(2 * MONITOR_INTERVAL)
        
        #       PRIMARY
        assertReadFrom(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        assertReadFrom(primary, PRIMARY_PREFERRED)

        #       SECONDARY
        assertReadFrom(other_secondary, SECONDARY)
        assertReadFrom(other_secondary, SECONDARY, self.other_secondary_dc)

        # Only the down secondary matches
        assertReadFrom(None, SECONDARY, {'name': 'secondary'})

        #       SECONDARY_PREFERRED
        assertReadFrom(other_secondary, SECONDARY_PREFERRED)
        assertReadFrom(
            other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)

        # The secondary matching the tag is down, use primary
        assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})

        #       NEAREST
        assertReadFromAll([primary, other_secondary], NEAREST)
        assertReadFrom(other_secondary, NEAREST, {'name': 'other_secondary'})
        assertReadFrom(primary, NEAREST, {'name': 'primary'})

        # 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
        replset_tools.kill_members([unpartition_node(other_secondary)], 2)
        self.assertTrue(Connection(
            unpartition_node(primary), use_greenlets=use_greenlets,
            slave_okay=True
        ).admin.command('ismaster')['ismaster'])

        #       PRIMARY
        assertReadFrom(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        assertReadFrom(primary, PRIMARY_PREFERRED)
        assertReadFrom(primary, PRIMARY_PREFERRED, self.secondary_dc)

        #       SECONDARY
        assertReadFrom(None, SECONDARY)
        assertReadFrom(None, SECONDARY, self.other_secondary_dc)
        assertReadFrom(None, SECONDARY, {'dc': 'ny'})

        #       SECONDARY_PREFERRED
        assertReadFrom(primary, SECONDARY_PREFERRED)
        assertReadFrom(primary, SECONDARY_PREFERRED, self.secondary_dc)
        assertReadFrom(primary, SECONDARY_PREFERRED, {'name': 'secondary'})
        assertReadFrom(primary, SECONDARY_PREFERRED, {'dc': 'ny'})

        #       NEAREST
        assertReadFrom(primary, NEAREST)
        assertReadFrom(None, NEAREST, self.secondary_dc)
        assertReadFrom(None, NEAREST, {'name': 'secondary'})

        # Even if primary's slow, still read from it
        self.set_ping_time(primary, 100)
        assertReadFrom(primary, NEAREST)
        assertReadFrom(None, NEAREST, self.secondary_dc)

        self.clear_ping_times()
    def test_read_preference(self):
        c = ReplicaSetConnection(self.seed, replicaSet=self.name)
        self.assertTrue(bool(len(c.secondaries)))
        db = c.pymongo_test
        db.test.remove({}, safe=True, w=len(c.secondaries))

        # Force replication...
        w = len(c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, safe=True, w=w)

        # Test direct connection to a secondary
        host, port = replset_tools.get_secondaries()[0].split(':')
        port = int(port)
        conn = Connection(host, port, slave_okay=True)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
        self.assert_(conn.pymongo_test.test.find_one())
        conn = Connection(host, port,
                          read_preference=ReadPreference.SECONDARY)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
        self.assert_(conn.pymongo_test.test.find_one())

        # Test direct connection to an arbiter
        host = replset_tools.get_arbiters()[0]
        self.assertRaises(ConnectionFailure, Connection, host)

        # Test PRIMARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertEqual(cursor._Cursor__connection_id, c.primary)

        # Test SECONDARY with a secondary
        db.read_preference = ReadPreference.SECONDARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertTrue(cursor._Cursor__connection_id in c.secondaries)

        # Test SECONDARY_ONLY with a secondary
        db.read_preference = ReadPreference.SECONDARY_ONLY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertTrue(cursor._Cursor__connection_id in c.secondaries)

        # Test SECONDARY with no secondary
        killed = replset_tools.kill_all_secondaries()
        self.assertTrue(bool(len(killed)))
        db.read_preference = ReadPreference.SECONDARY
        for _ in xrange(10):
            cursor = db.test.find()
            cursor.next()
            self.assertEqual(cursor._Cursor__connection_id, c.primary)

        # Test SECONDARY_ONLY with no secondary
        db.read_preference = ReadPreference.SECONDARY_ONLY
        for _ in xrange(10):
            cursor = db.test.find()
            self.assertRaises(AutoReconnect, cursor.next)

        replset_tools.restart_members(killed)
        # Test PRIMARY with no primary (should raise an exception)
        db.read_preference = ReadPreference.PRIMARY
        cursor = db.test.find()
        cursor.next()
        self.assertEqual(cursor._Cursor__connection_id, c.primary)
        killed = replset_tools.kill_primary()
        self.assertTrue(bool(len(killed)))
        self.assertRaises(AutoReconnect, db.test.find_one)