Ejemplo n.º 1
0
    def test_passive_and_hidden(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()

        passives = ha_tools.get_passives()
        passives = [_partition_node(member) for member in passives]
        self.assertEqual(self.c.secondaries, set(passives))

        for mode in SECONDARY, SECONDARY_PREFERRED:
            yield assert_read_from_all(self, self.c, passives, mode)

        ha_tools.kill_members(ha_tools.get_passives(), 2)
        yield self.pause(2 * MONITOR_INTERVAL)
        yield assert_read_from(
            self, self.c, self.c.primary, SECONDARY_PREFERRED)
Ejemplo n.º 2
0
    def test_passive_and_hidden(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()

        passives = ha_tools.get_passives()
        passives = [_partition_node(member) for member in passives]
        self.assertEqual(self.c.secondaries, set(passives))

        for mode in SECONDARY, SECONDARY_PREFERRED:
            yield assert_read_from_all(self, self.c, passives, mode)

        ha_tools.kill_members(ha_tools.get_passives(), 2)
        yield self.pause(2 * MONITOR_INTERVAL)
        yield assert_read_from(self, self.c, self.c.primary,
                               SECONDARY_PREFERRED)
Ejemplo n.º 3
0
    def test_monitor_removes_recovering_member(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()
        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            yield assert_read_from_all(
                self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        yield self.pause(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            yield assert_read_from(
                self, self.c, _partition_node(secondary), mode)
Ejemplo n.º 4
0
    def test_monitor_removes_recovering_member(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()
        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            yield assert_read_from_all(self, self.c, partitioned_secondaries,
                                       mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        yield self.pause(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            yield assert_read_from(self, self.c, _partition_node(secondary),
                                   mode)
Ejemplo n.º 5
0
    def test_read_preference(self):
        # This is long, but we put all the tests in one function to save time
        # on setUp, which takes about 30 seconds to bring up a replica set.
        # We pass through four states:
        #
        #       1. A primary and two secondaries
        #       2. Primary down
        #       3. Primary up, one secondary down
        #       4. Primary up, all secondaries down
        #
        # For each state, we verify the behavior of PRIMARY,
        # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
        c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield c.open()

        @gen.coroutine
        def read_from_which_host(
            rsc,
            mode,
            tag_sets=None,
            latency=15,
        ):
            db = rsc.motor_test
            db.read_preference = mode
            if isinstance(tag_sets, dict):
                tag_sets = [tag_sets]
            db.tag_sets = tag_sets or [{}]
            db.secondary_acceptable_latency_ms = latency

            cursor = db.test.find()
            try:
                yield cursor.fetch_next
                raise gen.Return(cursor.delegate._Cursor__connection_id)
            except AutoReconnect:
                raise gen.Return(None)

        @gen.coroutine
        def assert_read_from(member, *args, **kwargs):
            for _ in range(10):
                used = yield read_from_which_host(c, *args, **kwargs)
                self.assertEqual(member, used)

        @gen.coroutine
        def assert_read_from_all(members, *args, **kwargs):
            members = set(members)
            all_used = set()
            for _ in range(100):
                used = yield read_from_which_host(c, *args, **kwargs)
                all_used.add(used)
                if members == all_used:
                    raise gen.Return()  # Success

            # This will fail
            self.assertEqual(members, all_used)

        def unpartition_node(node):
            host, port = node
            return '%s:%s' % (host, port)

        primary = self.primary
        secondary = self.secondary
        other_secondary = self.other_secondary

        bad_tag = {'bad': 'tag'}

        # 1. THREE MEMBERS UP -------------------------------------------------
        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        # Trivial: mode and tags both match
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Secondary matches but not primary, choose primary
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)

        # Chooses primary, ignoring tag sets
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Chooses primary, ignoring tag sets
        yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
        yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])

        #       SECONDARY
        yield assert_read_from_all(
            [secondary, other_secondary], SECONDARY, latency=9999999)

        #       SECONDARY_PREFERRED
        yield assert_read_from_all(
            [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)

        # Multiple tags
        yield assert_read_from(
            secondary, SECONDARY_PREFERRED, self.secondary_tags)

        # Fall back to primary if it's the only one matching the tags
        yield assert_read_from(
            primary, SECONDARY_PREFERRED, {'name': 'primary'})

        # No matching secondaries
        yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)

        # Fall back from non-matching tag set to matching set
        yield assert_read_from_all(
            [secondary, other_secondary],
            SECONDARY_PREFERRED, [bad_tag, {}], latency=9999999)

        yield assert_read_from(
            other_secondary,
            SECONDARY_PREFERRED, [bad_tag, {'dc': 'ny'}])

        #       NEAREST
        self.clear_ping_times()

        yield assert_read_from_all(
            [primary, secondary, other_secondary], NEAREST, latency=9999999)

        yield assert_read_from_all(
            [primary, other_secondary],
            NEAREST, [bad_tag, {'dc': 'ny'}], latency=9999999)

        self.set_ping_time(primary, 0)
        self.set_ping_time(secondary, .03)  # 30 milliseconds.
        self.set_ping_time(other_secondary, 10)

        # Nearest member, no tags
        yield assert_read_from(primary, NEAREST)

        # Tags override nearness
        yield assert_read_from(primary, NEAREST, {'name': 'primary'})
        yield assert_read_from(secondary, NEAREST, self.secondary_dc)

        # Make secondary fast
        self.set_ping_time(primary, .03)  # 30 milliseconds.
        self.set_ping_time(secondary, 0)

        yield assert_read_from(secondary, NEAREST)

        # Other secondary fast
        self.set_ping_time(secondary, 10)
        self.set_ping_time(other_secondary, 0)

        yield assert_read_from(other_secondary, NEAREST)

        # High secondaryAcceptableLatencyMS, should read from all members
        yield assert_read_from_all(
            [primary, secondary, other_secondary],
            NEAREST, latency=9999999)

        self.clear_ping_times()

        yield assert_read_from_all(
            [primary, other_secondary], NEAREST, [{'dc': 'ny'}],
            latency=9999999)

        # 2. PRIMARY DOWN -----------------------------------------------------
        killed = ha_tools.kill_primary()

        # Let monitor notice primary's gone
        yield self.pause(2 * MONITOR_INTERVAL)

        #       PRIMARY
        yield assert_read_from(None, PRIMARY)

        #       PRIMARY_PREFERRED
        # No primary, choose matching secondary
        yield assert_read_from_all(
            [secondary, other_secondary], PRIMARY_PREFERRED, latency=9999999)

        yield assert_read_from(
            secondary, PRIMARY_PREFERRED, {'name': 'secondary'})

        # No primary or matching secondary
        yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)

        #       SECONDARY
        yield assert_read_from_all(
            [secondary, other_secondary], SECONDARY, latency=9999999)

        # Only primary matches
        yield assert_read_from(None, SECONDARY, {'name': 'primary'})

        # No matching secondaries
        yield assert_read_from(None, SECONDARY, bad_tag)

        #       SECONDARY_PREFERRED
        yield assert_read_from_all(
            [secondary, other_secondary], SECONDARY_PREFERRED, latency=9999999)

        # Mode and tags both match
        yield assert_read_from(
            secondary, SECONDARY_PREFERRED, {'name': 'secondary'})

        #       NEAREST
        self.clear_ping_times()

        yield assert_read_from_all(
            [secondary, other_secondary], NEAREST, latency=9999999)

        # 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
        ha_tools.restart_members([killed])

        for _ in range(30):
            if ha_tools.get_primary():
                break
            yield self.pause(1)
        else:
            self.fail("Primary didn't come back up")

        ha_tools.kill_members([unpartition_node(secondary)], 2)
        self.assertTrue(pymongo.MongoClient(
            unpartition_node(primary),
            slave_okay=True
        ).admin.command('ismaster')['ismaster'])

        yield self.pause(2 * MONITOR_INTERVAL)

        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        yield assert_read_from(primary, PRIMARY_PREFERRED)

        #       SECONDARY
        yield assert_read_from(other_secondary, SECONDARY)
        yield assert_read_from(
            other_secondary, SECONDARY, self.other_secondary_dc)

        # Only the down secondary matches
        yield assert_read_from(None, SECONDARY, {'name': 'secondary'})

        #       SECONDARY_PREFERRED
        yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
        yield assert_read_from(
            other_secondary, SECONDARY_PREFERRED, self.other_secondary_dc)

        # The secondary matching the tag is down, use primary
        yield assert_read_from(
            primary, SECONDARY_PREFERRED, {'name': 'secondary'})

        #       NEAREST
        yield assert_read_from_all(
            [primary, other_secondary], NEAREST, latency=9999999)

        yield assert_read_from(
            other_secondary, NEAREST, {'name': 'other_secondary'})

        yield assert_read_from(primary, NEAREST, {'name': 'primary'})

        # 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
        ha_tools.kill_members([unpartition_node(other_secondary)], 2)
        self.assertTrue(pymongo.MongoClient(
            unpartition_node(primary),
            slave_okay=True
        ).admin.command('ismaster')['ismaster'])

        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        yield assert_read_from(primary, PRIMARY_PREFERRED)
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)

        #       SECONDARY
        yield assert_read_from(None, SECONDARY)
        yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
        yield assert_read_from(None, SECONDARY, {'dc': 'ny'})

        #       SECONDARY_PREFERRED
        yield assert_read_from(primary, SECONDARY_PREFERRED)
        yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
        yield assert_read_from(
            primary, SECONDARY_PREFERRED, {'name': 'secondary'})

        yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})

        #       NEAREST
        yield assert_read_from(primary, NEAREST)
        yield assert_read_from(None, NEAREST, self.secondary_dc)
        yield assert_read_from(None, NEAREST, {'name': 'secondary'})

        # Even if primary's slow, still read from it
        self.set_ping_time(primary, 100)
        yield assert_read_from(primary, NEAREST)
        yield assert_read_from(None, NEAREST, self.secondary_dc)

        self.clear_ping_times()
Ejemplo n.º 6
0
    def test_read_preference(self):
        # This is long, but we put all the tests in one function to save time
        # on setUp, which takes about 30 seconds to bring up a replica set.
        # We pass through four states:
        #
        #       1. A primary and two secondaries
        #       2. Primary down
        #       3. Primary up, one secondary down
        #       4. Primary up, all secondaries down
        #
        # For each state, we verify the behavior of PRIMARY,
        # PRIMARY_PREFERRED, SECONDARY, SECONDARY_PREFERRED, and NEAREST
        c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield c.open()

        @gen.coroutine
        def read_from_which_host(
            rsc,
            mode,
            tag_sets=None,
            latency=15,
        ):
            db = rsc.motor_test
            db.read_preference = mode
            if isinstance(tag_sets, dict):
                tag_sets = [tag_sets]
            db.tag_sets = tag_sets or [{}]
            db.secondary_acceptable_latency_ms = latency

            cursor = db.test.find()
            try:
                yield cursor.fetch_next
                raise gen.Return(cursor.delegate._Cursor__connection_id)
            except AutoReconnect:
                raise gen.Return(None)

        @gen.coroutine
        def assert_read_from(member, *args, **kwargs):
            for _ in range(10):
                used = yield read_from_which_host(c, *args, **kwargs)
                self.assertEqual(member, used)

        @gen.coroutine
        def assert_read_from_all(members, *args, **kwargs):
            members = set(members)
            all_used = set()
            for _ in range(100):
                used = yield read_from_which_host(c, *args, **kwargs)
                all_used.add(used)
                if members == all_used:
                    raise gen.Return()  # Success

            # This will fail
            self.assertEqual(members, all_used)

        def unpartition_node(node):
            host, port = node
            return '%s:%s' % (host, port)

        primary = self.primary
        secondary = self.secondary
        other_secondary = self.other_secondary

        bad_tag = {'bad': 'tag'}

        # 1. THREE MEMBERS UP -------------------------------------------------
        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        # Trivial: mode and tags both match
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Secondary matches but not primary, choose primary
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)

        # Chooses primary, ignoring tag sets
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.primary_dc)

        # Chooses primary, ignoring tag sets
        yield assert_read_from(primary, PRIMARY_PREFERRED, bad_tag)
        yield assert_read_from(primary, PRIMARY_PREFERRED, [bad_tag, {}])

        #       SECONDARY
        yield assert_read_from_all([secondary, other_secondary],
                                   SECONDARY,
                                   latency=9999999)

        #       SECONDARY_PREFERRED
        yield assert_read_from_all([secondary, other_secondary],
                                   SECONDARY_PREFERRED,
                                   latency=9999999)

        # Multiple tags
        yield assert_read_from(secondary, SECONDARY_PREFERRED,
                               self.secondary_tags)

        # Fall back to primary if it's the only one matching the tags
        yield assert_read_from(primary, SECONDARY_PREFERRED,
                               {'name': 'primary'})

        # No matching secondaries
        yield assert_read_from(primary, SECONDARY_PREFERRED, bad_tag)

        # Fall back from non-matching tag set to matching set
        yield assert_read_from_all([secondary, other_secondary],
                                   SECONDARY_PREFERRED, [bad_tag, {}],
                                   latency=9999999)

        yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
                               [bad_tag, {
                                   'dc': 'ny'
                               }])

        #       NEAREST
        self.clear_ping_times()

        yield assert_read_from_all([primary, secondary, other_secondary],
                                   NEAREST,
                                   latency=9999999)

        yield assert_read_from_all([primary, other_secondary],
                                   NEAREST, [bad_tag, {
                                       'dc': 'ny'
                                   }],
                                   latency=9999999)

        self.set_ping_time(primary, 0)
        self.set_ping_time(secondary, .03)  # 30 milliseconds.
        self.set_ping_time(other_secondary, 10)

        # Nearest member, no tags
        yield assert_read_from(primary, NEAREST)

        # Tags override nearness
        yield assert_read_from(primary, NEAREST, {'name': 'primary'})
        yield assert_read_from(secondary, NEAREST, self.secondary_dc)

        # Make secondary fast
        self.set_ping_time(primary, .03)  # 30 milliseconds.
        self.set_ping_time(secondary, 0)

        yield assert_read_from(secondary, NEAREST)

        # Other secondary fast
        self.set_ping_time(secondary, 10)
        self.set_ping_time(other_secondary, 0)

        yield assert_read_from(other_secondary, NEAREST)

        # High secondaryAcceptableLatencyMS, should read from all members
        yield assert_read_from_all([primary, secondary, other_secondary],
                                   NEAREST,
                                   latency=9999999)

        self.clear_ping_times()

        yield assert_read_from_all([primary, other_secondary],
                                   NEAREST, [{
                                       'dc': 'ny'
                                   }],
                                   latency=9999999)

        # 2. PRIMARY DOWN -----------------------------------------------------
        killed = ha_tools.kill_primary()

        # Let monitor notice primary's gone
        yield self.pause(2 * MONITOR_INTERVAL)

        #       PRIMARY
        yield assert_read_from(None, PRIMARY)

        #       PRIMARY_PREFERRED
        # No primary, choose matching secondary
        yield assert_read_from_all([secondary, other_secondary],
                                   PRIMARY_PREFERRED,
                                   latency=9999999)

        yield assert_read_from(secondary, PRIMARY_PREFERRED,
                               {'name': 'secondary'})

        # No primary or matching secondary
        yield assert_read_from(None, PRIMARY_PREFERRED, bad_tag)

        #       SECONDARY
        yield assert_read_from_all([secondary, other_secondary],
                                   SECONDARY,
                                   latency=9999999)

        # Only primary matches
        yield assert_read_from(None, SECONDARY, {'name': 'primary'})

        # No matching secondaries
        yield assert_read_from(None, SECONDARY, bad_tag)

        #       SECONDARY_PREFERRED
        yield assert_read_from_all([secondary, other_secondary],
                                   SECONDARY_PREFERRED,
                                   latency=9999999)

        # Mode and tags both match
        yield assert_read_from(secondary, SECONDARY_PREFERRED,
                               {'name': 'secondary'})

        #       NEAREST
        self.clear_ping_times()

        yield assert_read_from_all([secondary, other_secondary],
                                   NEAREST,
                                   latency=9999999)

        # 3. PRIMARY UP, ONE SECONDARY DOWN -----------------------------------
        ha_tools.restart_members([killed])

        for _ in range(30):
            if ha_tools.get_primary():
                break
            yield self.pause(1)
        else:
            self.fail("Primary didn't come back up")

        ha_tools.kill_members([unpartition_node(secondary)], 2)
        self.assertTrue(
            pymongo.MongoClient(
                unpartition_node(primary),
                slave_okay=True).admin.command('ismaster')['ismaster'])

        yield self.pause(2 * MONITOR_INTERVAL)

        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        yield assert_read_from(primary, PRIMARY_PREFERRED)

        #       SECONDARY
        yield assert_read_from(other_secondary, SECONDARY)
        yield assert_read_from(other_secondary, SECONDARY,
                               self.other_secondary_dc)

        # Only the down secondary matches
        yield assert_read_from(None, SECONDARY, {'name': 'secondary'})

        #       SECONDARY_PREFERRED
        yield assert_read_from(other_secondary, SECONDARY_PREFERRED)
        yield assert_read_from(other_secondary, SECONDARY_PREFERRED,
                               self.other_secondary_dc)

        # The secondary matching the tag is down, use primary
        yield assert_read_from(primary, SECONDARY_PREFERRED,
                               {'name': 'secondary'})

        #       NEAREST
        yield assert_read_from_all([primary, other_secondary],
                                   NEAREST,
                                   latency=9999999)

        yield assert_read_from(other_secondary, NEAREST,
                               {'name': 'other_secondary'})

        yield assert_read_from(primary, NEAREST, {'name': 'primary'})

        # 4. PRIMARY UP, ALL SECONDARIES DOWN ---------------------------------
        ha_tools.kill_members([unpartition_node(other_secondary)], 2)
        self.assertTrue(
            pymongo.MongoClient(
                unpartition_node(primary),
                slave_okay=True).admin.command('ismaster')['ismaster'])

        #       PRIMARY
        yield assert_read_from(primary, PRIMARY)

        #       PRIMARY_PREFERRED
        yield assert_read_from(primary, PRIMARY_PREFERRED)
        yield assert_read_from(primary, PRIMARY_PREFERRED, self.secondary_dc)

        #       SECONDARY
        yield assert_read_from(None, SECONDARY)
        yield assert_read_from(None, SECONDARY, self.other_secondary_dc)
        yield assert_read_from(None, SECONDARY, {'dc': 'ny'})

        #       SECONDARY_PREFERRED
        yield assert_read_from(primary, SECONDARY_PREFERRED)
        yield assert_read_from(primary, SECONDARY_PREFERRED, self.secondary_dc)
        yield assert_read_from(primary, SECONDARY_PREFERRED,
                               {'name': 'secondary'})

        yield assert_read_from(primary, SECONDARY_PREFERRED, {'dc': 'ny'})

        #       NEAREST
        yield assert_read_from(primary, NEAREST)
        yield assert_read_from(None, NEAREST, self.secondary_dc)
        yield assert_read_from(None, NEAREST, {'name': 'secondary'})

        # Even if primary's slow, still read from it
        self.set_ping_time(primary, 100)
        yield assert_read_from(primary, NEAREST)
        yield assert_read_from(None, NEAREST, self.secondary_dc)

        self.clear_ping_times()