예제 #1
0
    def test_ship_of_theseus(self):
        c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        db = c.pymongo_test
        db.test.insert({}, w=len(c.secondaries) + 1)
        find_one = db.test.find_one

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()

        new_hosts = [ha_tools.add_member() for _ in range(3)]

        # Wait for new members to join.
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            sleep(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)

        # Wait for primary.
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            sleep(1)
        else:
            self.fail("No failover")

        sleep(2 * MONITOR_INTERVAL)

        # No error.
        find_one()
        find_one(read_preference=SECONDARY)

        # All members down.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members(new_hosts)

        # Should be able to reconnect to set even though original seed
        # list is useless. Use SECONDARY so we don't have to wait for
        # the election, merely for the client to detect members are up.
        sleep(2 * MONITOR_INTERVAL)
        find_one(read_preference=SECONDARY)
예제 #2
0
    def test_ship_of_theseus(self, done):
        loop = IOLoop.instance()
        c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        c.open_sync()

        db = c.pymongo_test
        w = len(c.secondaries) + 1
        db.test.insert({}, w=w)

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()
        ha_tools.add_member()
        ha_tools.add_member()
        ha_tools.add_member()

        # Wait for new members to join
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            yield gen.Task(loop.add_timeout, time.time() + 1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)

        # Wait for primary
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            yield gen.Task(loop.add_timeout, time.time() + 1)
        else:
            self.fail("No failover")

        # Ensure monitor picks up new members
        yield gen.Task(loop.add_timeout, time.time() + 2 * MONITOR_INTERVAL)

        try:
            yield motor.Op(db.test.find_one)
        except AutoReconnect:
            # Might take one try to reconnect
            yield gen.Task(loop.add_timeout, time.time() + 1)

        # No error
        yield motor.Op(db.test.find_one)
        yield motor.Op(db.test.find_one, read_preference=SECONDARY)

        done()
예제 #3
0
    def test_stepdown_triggers_refresh(self):
        c_find_one = MongoReplicaSetClient(self.seed,
                                           replicaSet=self.name,
                                           use_greenlets=use_greenlets)

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]
        self.assertEqual(one(c_find_one.secondaries),
                         _partition_node(secondary))

        ha_tools.stepdown_primary()

        # Make sure the stepdown completes
        sleep(1)

        # Trigger a refresh
        self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        sleep(1)

        # We've detected the stepdown
        self.assertTrue(not c_find_one.primary
                        or primary != _partition_node(c_find_one.primary))
예제 #4
0
    def test_recovering_member_triggers_refresh(self):
        # To test that find_one() and count() trigger immediate refreshes,
        # we'll create a separate client for each
        self.c_find_one, self.c_count = yield [
            motor.MotorReplicaSetClient(self.seed,
                                        replicaSet=self.name,
                                        read_preference=SECONDARY).open()
            for _ in xrange(2)
        ]

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]

        # Pre-condition: just make sure they all connected OK
        for c in self.c_find_one, self.c_count:
            self.assertEqual(one(c.secondaries), _partition_node(secondary))

        ha_tools.set_maintenance(secondary, True)

        # Trigger a refresh in various ways
        with assert_raises(AutoReconnect):
            yield self.c_find_one.test.test.find_one()

        with assert_raises(AutoReconnect):
            yield self.c_count.test.test.count()

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        yield self.pause(1)

        for c in self.c_find_one, self.c_count:
            self.assertFalse(c.secondaries)
            self.assertEqual(_partition_node(primary), c.primary)
예제 #5
0
    def test_stepdown_triggers_refresh(self):
        c_find_one = yield motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name).open()

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]
        self.assertEqual(one(c_find_one.secondaries),
                         _partition_node(secondary))

        ha_tools.stepdown_primary()

        # Make sure the stepdown completes
        yield self.pause(1)

        # Trigger a refresh
        with assert_raises(AutoReconnect):
            yield c_find_one.test.test.find_one()

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        yield self.pause(1)

        # We've detected the stepdown
        self.assertTrue(not c_find_one.primary
                        or primary != _partition_node(c_find_one.primary))
예제 #6
0
    def test_stepdown_triggers_refresh(self, done):
        c_find_one = motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name).open_sync()

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]
        self.assertEqual(
            one(c_find_one.secondaries), _partition_node(secondary))

        ha_tools.stepdown_primary()

        # Make sure the stepdown completes
        yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)

        # Trigger a refresh
        yield AssertRaises(AutoReconnect, c_find_one.test.test.find_one)

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        yield gen.Task(IOLoop.instance().add_timeout, time.time() + 1)

        # We've detected the stepdown
        self.assertTrue(
            not c_find_one.primary
            or primary != _partition_node(c_find_one.primary))

        done()
예제 #7
0
    def test_stepdown_triggers_refresh(self):
        c_find_one = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]
        self.assertEqual(
            one(c_find_one.secondaries), _partition_node(secondary))

        ha_tools.stepdown_primary()

        # Make sure the stepdown completes
        sleep(1)

        # Trigger a refresh
        self.assertRaises(AutoReconnect, c_find_one.test.test.find_one)

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        sleep(1)

        # We've detected the stepdown
        self.assertTrue(
            not c_find_one.primary
            or _partition_node(primary) != c_find_one.primary)
예제 #8
0
    def test_recovering_member_triggers_refresh(self):
        # To test that find_one() and count() trigger immediate refreshes,
        # we'll create a separate client for each
        self.c_find_one, self.c_count = yield [
            motor.MotorReplicaSetClient(
                self.seed, replicaSet=self.name, read_preference=SECONDARY
            ).open() for _ in xrange(2)]

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]

        # Pre-condition: just make sure they all connected OK
        for c in self.c_find_one, self.c_count:
            self.assertEqual(one(c.secondaries), _partition_node(secondary))

        ha_tools.set_maintenance(secondary, True)

        # Trigger a refresh in various ways
        with assert_raises(AutoReconnect):
            yield self.c_find_one.test.test.find_one()

        with assert_raises(AutoReconnect):
            yield self.c_count.test.test.count()

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        yield self.pause(1)

        for c in self.c_find_one, self.c_count:
            self.assertFalse(c.secondaries)
            self.assertEqual(_partition_node(primary), c.primary)
예제 #9
0
    def test_stepdown_triggers_refresh(self):
        c_find_one = yield motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name).open()

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]
        self.assertEqual(
            one(c_find_one.secondaries), _partition_node(secondary))

        ha_tools.stepdown_primary()

        # Make sure the stepdown completes
        yield self.pause(1)

        # Trigger a refresh
        with assert_raises(AutoReconnect):
            yield c_find_one.test.test.find_one()

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        yield self.pause(1)

        # We've detected the stepdown
        self.assertTrue(
            not c_find_one.primary
            or primary != _partition_node(c_find_one.primary))
예제 #10
0
    def test_primary_stepdown(self):
        c = MongoReplicaSetClient(self.seed,
                                  replicaSet=self.name,
                                  use_greenlets=use_greenlets)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        ha_tools.stepdown_primary()

        # Wait for new primary
        patience_seconds = 30
        for _ in xrange(patience_seconds):
            sleep(1)
            rs_state = c._MongoReplicaSetClient__rs_state
            if rs_state.writer and rs_state.writer != primary:
                if ha_tools.get_primary():
                    # New primary stepped up
                    new_primary = _partition_node(ha_tools.get_primary())
                    self.assertEqual(new_primary, rs_state.writer)
                    new_secondaries = partition_nodes(
                        ha_tools.get_secondaries())
                    self.assertEqual(set(new_secondaries),
                                     rs_state.secondaries)
                    break
        else:
            self.fail(
                "No new primary after %s seconds. Old primary was %s, current"
                " is %s" % (patience_seconds, primary, ha_tools.get_primary()))
예제 #11
0
    def test_secondary_connection(self):
        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.pymongo_test
        db.test.remove({}, safe=True, w=len(self.c.secondaries))

        # Force replication...
        w = len(self.c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, safe=True, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)

        self.assertTrue(Connection(
            primary_host, primary_port, use_greenlets=use_greenlets).is_primary)

        self.assertTrue(Connection(
            primary_host, primary_port, use_greenlets=use_greenlets,
            read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)

        self.assertTrue(Connection(
            primary_host, primary_port, use_greenlets=use_greenlets,
            read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)

        self.assertTrue(Connection(
            primary_host, primary_port, use_greenlets=use_greenlets,
            read_preference=ReadPreference.NEAREST).is_primary)

        self.assertTrue(Connection(
            primary_host, primary_port, use_greenlets=use_greenlets,
            read_preference=ReadPreference.SECONDARY).is_primary)

        for kwargs in [
            {'read_preference': ReadPreference.PRIMARY_PREFERRED},
            {'read_preference': ReadPreference.SECONDARY},
            {'read_preference': ReadPreference.SECONDARY_PREFERRED},
            {'read_preference': ReadPreference.NEAREST},
            {'slave_okay': True},
        ]:
            conn = Connection(secondary_host,
                              secondary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(secondary_host, conn.host)
            self.assertEqual(secondary_port, conn.port)
            self.assertFalse(conn.is_primary)
            self.assert_(conn.pymongo_test.test.find_one())

        # Test direct connection to an arbiter
        secondary_host = ha_tools.get_arbiters()[0]
        host, port = ha_tools.get_arbiters()[0].split(':')
        port = int(port)
        conn = Connection(host, port)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
예제 #12
0
    def test_ship_of_theseus(self):
        c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield c.open()
        db = c.motor_test
        w = len(c.secondaries) + 1
        db.test.insert({}, w=w)

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()
        ha_tools.add_member()
        ha_tools.add_member()
        ha_tools.add_member()

        # Wait for new members to join
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            yield self.pause(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)

        # Wait for primary
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            yield self.pause(1)
        else:
            self.fail("No failover")

        # Ensure monitor picks up new members
        yield self.pause(2 * MONITOR_INTERVAL)

        try:
            yield db.test.find_one()
        except AutoReconnect:
            # Might take one try to reconnect
            yield self.pause(1)

        # No error
        yield db.test.find_one()
        yield db.test.find_one(read_preference=SECONDARY)
예제 #13
0
    def test_ship_of_theseus(self):
        c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield c.open()
        db = c.motor_test
        w = len(c.secondaries) + 1
        db.test.insert({}, w=w)

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()
        ha_tools.add_member()
        ha_tools.add_member()
        ha_tools.add_member()

        # Wait for new members to join
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            yield self.pause(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)

        # Wait for primary
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            yield self.pause(1)
        else:
            self.fail("No failover")

        # Ensure monitor picks up new members
        yield self.pause(2 * MONITOR_INTERVAL)

        try:
            yield db.test.find_one()
        except AutoReconnect:
            # Might take one try to reconnect
            yield self.pause(1)

        # No error
        yield db.test.find_one()
        yield db.test.find_one(read_preference=SECONDARY)
예제 #14
0
    def setUp(self):
        super(TestReadPreference, self).setUp()

        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        self.w = len(self.c.secondaries) + 1
        self.db = self.c.get_database("pymongo_test",
                                      write_concern=WriteConcern(w=self.w))
        self.db.test.delete_many({})
        self.db.test.insert_many([{'foo': i} for i in xrange(10)])

        self.clear_ping_times()
예제 #15
0
    def setUp(self):
        super(TestReadPreference, self).setUp()

        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        self.w = len(self.c.secondaries) + 1
        self.db = self.c.get_database("pymongo_test",
                                      write_concern=WriteConcern(w=self.w))
        self.db.test.delete_many({})
        self.db.test.insert_many([{'foo': i} for i in xrange(10)])

        self.clear_ping_times()
예제 #16
0
    def setUp(self):
        super(MotorTestReadPreference, self).setUp()
        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = _partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = _partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = _partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        # Synchronous PyMongo interfaces for convenience
        self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
            self.seed, replicaSet=self.name)
        self.db = self.c.pymongo_test
        self.w = len(self.c.secondaries) + 1
        self.db.test.remove({}, w=self.w)
        self.db.test.insert(
            [{'foo': i} for i in xrange(10)], w=self.w)

        self.clear_ping_times()
예제 #17
0
    def setUp(self):
        super(MotorTestReadPreference, self).setUp()
        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = _partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = _partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = _partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        # Synchronous PyMongo interfaces for convenience
        self.c = pymongo.mongo_replica_set_client.MongoReplicaSetClient(
            self.seed, replicaSet=self.name)
        self.db = self.c.motor_test
        self.w = len(self.c.secondaries) + 1
        self.db.test.remove({}, w=self.w)
        self.db.test.insert(
            [{'foo': i} for i in xrange(10)], w=self.w)

        self.clear_ping_times()
예제 #18
0
    def setUp(self):
        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = _partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = _partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = _partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.db = self.c.pymongo_test
        self.w = len(self.c.secondaries) + 1
        self.db.test.remove({}, safe=True, w=self.w)
        self.db.test.insert(
            [{'foo': i} for i in xrange(10)], safe=True, w=self.w)

        self.clear_ping_times()
예제 #19
0
    def setUp(self):
        members = [
            # primary
            {'tags': {'dc': 'ny', 'name': 'primary'}},

            # secondary
            {'tags': {'dc': 'la', 'name': 'secondary'}, 'priority': 0},

            # other_secondary
            {'tags': {'dc': 'ny', 'name': 'other_secondary'}, 'priority': 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = _partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual('primary', self.primary_tags['name'])

        self.primary_dc = {'dc': self.primary_tags['dc']}

        secondaries = ha_tools.get_secondaries()

        (secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'secondary']

        self.secondary = _partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {'dc': self.secondary_tags['dc']}

        (other_secondary, ) = [
            s for s in secondaries
            if ha_tools.get_tags(s)['name'] == 'other_secondary']

        self.other_secondary = _partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {'dc': self.other_secondary_tags['dc']}

        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.db = self.c.pymongo_test
        self.w = len(self.c.secondaries) + 1
        self.db.test.remove({}, safe=True, w=self.w)
        self.db.test.insert(
            [{'foo': i} for i in xrange(10)], safe=True, w=self.w)

        self.clear_ping_times()
예제 #20
0
    def test_monitor_removes_recovering_member(self):
        self.c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = partition_nodes(secondaries)
            utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        sleep(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
예제 #21
0
    def test_monitor_removes_recovering_member(self):
        self.c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = partition_nodes(secondaries)
            utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        sleep(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
예제 #22
0
    def test_monitor_removes_recovering_member(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()
        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            yield assert_read_from_all(self, self.c, partitioned_secondaries,
                                       mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        yield self.pause(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            yield assert_read_from(self, self.c, _partition_node(secondary),
                                   mode)
예제 #23
0
    def test_monitor_removes_recovering_member(self):
        self.c = motor.MotorReplicaSetClient(self.seed, replicaSet=self.name)
        yield self.c.open()
        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            yield assert_read_from_all(
                self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        yield self.pause(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            yield assert_read_from(
                self, self.c, _partition_node(secondary), mode)
예제 #24
0
    def test_recovering(self):
        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets,
            auto_start_request=False)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        sleep(2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            utils.assertReadFrom(self, self.c, _partition_node(secondary), mode)
예제 #25
0
    def test_monitor_removes_recovering_member(self):
        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = partition_nodes(secondaries)
            utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        time.sleep(2 * self.heartbeat_frequency)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            utils.assertReadFrom(self, self.c, partition_node(secondary), mode)
예제 #26
0
    def test_monitor_removes_recovering_member(self):
        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = partition_nodes(secondaries)
            utils.assertReadFromAll(self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        time.sleep(2 * self.heartbeat_frequency)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            utils.assertReadFrom(self, self.c, partition_node(secondary), mode)
    def setUp(self):
        members = [
            # primary
            {"tags": {"dc": "ny", "name": "primary"}},
            # secondary
            {"tags": {"dc": "la", "name": "secondary"}, "priority": 0},
            # other_secondary
            {"tags": {"dc": "ny", "name": "other_secondary"}, "priority": 0},
        ]

        res = ha_tools.start_replica_set(members)
        self.seed, self.name = res

        primary = ha_tools.get_primary()
        self.primary = _partition_node(primary)
        self.primary_tags = ha_tools.get_tags(primary)
        # Make sure priority worked
        self.assertEqual("primary", self.primary_tags["name"])

        self.primary_dc = {"dc": self.primary_tags["dc"]}

        secondaries = ha_tools.get_secondaries()

        (secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "secondary"]

        self.secondary = _partition_node(secondary)
        self.secondary_tags = ha_tools.get_tags(secondary)
        self.secondary_dc = {"dc": self.secondary_tags["dc"]}

        (other_secondary,) = [s for s in secondaries if ha_tools.get_tags(s)["name"] == "other_secondary"]

        self.other_secondary = _partition_node(other_secondary)
        self.other_secondary_tags = ha_tools.get_tags(other_secondary)
        self.other_secondary_dc = {"dc": self.other_secondary_tags["dc"]}

        self.c = ReplicaSetConnection(self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.db = self.c.pymongo_test
        self.w = len(self.c.secondaries) + 1
        self.db.test.remove({}, safe=True, w=self.w)
        self.db.test.insert([{"foo": i} for i in xrange(10)], safe=True, w=self.w)

        self.clear_ping_times()
예제 #28
0
    def test_monitor_removes_recovering_member(self):
        self.c = motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name)

        secondaries = ha_tools.get_secondaries()

        for mode in SECONDARY, SECONDARY_PREFERRED:
            partitioned_secondaries = [_partition_node(s) for s in secondaries]
            yield motor.Op(assertReadFromAll,
                self, self.c, partitioned_secondaries, mode)

        secondary, recovering_secondary = secondaries
        ha_tools.set_maintenance(recovering_secondary, True)
        yield gen.Task(
            IOLoop.instance().add_timeout, time.time() + 2 * MONITOR_INTERVAL)

        for mode in SECONDARY, SECONDARY_PREFERRED:
            # Don't read from recovering member
            yield motor.Op(assertReadFrom,
                self, self.c, _partition_node(secondary), mode)
예제 #29
0
    def test_recovering_member_triggers_refresh(self):
        # To test that find_one() and count() trigger immediate refreshes,
        # we'll create a separate client for each
        self.c_find_one, self.c_count = [
            MongoClient(
                self.seed,
                replicaSet=self.name,
                read_preference=SECONDARY,
                serverSelectionTimeoutMS=self.server_selection_timeout)
            for _ in xrange(2)]

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]

        # Pre-condition: just make sure they all connected OK
        for c in self.c_find_one, self.c_count:
            wait_until(
                lambda: c.primary == partition_node(primary),
                'connect to the primary')

            wait_until(
                lambda: one(c.secondaries) == partition_node(secondary),
                'connect to the secondary')

        ha_tools.set_maintenance(secondary, True)

        # Trigger a refresh in various ways
        self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
        self.assertRaises(AutoReconnect, self.c_count.test.test.count)

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        time.sleep(1)

        self.assertFalse(self.c_find_one.secondaries)
        self.assertEqual(partition_node(primary), self.c_find_one.primary)

        self.assertFalse(self.c_count.secondaries)
        self.assertEqual(partition_node(primary), self.c_count.primary)
예제 #30
0
    def test_recovering_member_triggers_refresh(self):
        # To test that find_one() and count() trigger immediate refreshes,
        # we'll create a separate client for each
        self.c_find_one, self.c_count = [
            MongoClient(
                self.seed,
                replicaSet=self.name,
                read_preference=SECONDARY,
                serverSelectionTimeoutMS=self.server_selection_timeout)
            for _ in xrange(2)]

        # We've started the primary and one secondary
        primary = ha_tools.get_primary()
        secondary = ha_tools.get_secondaries()[0]

        # Pre-condition: just make sure they all connected OK
        for c in self.c_find_one, self.c_count:
            wait_until(
                lambda: c.primary == partition_node(primary),
                'connect to the primary')

            wait_until(
                lambda: one(c.secondaries) == partition_node(secondary),
                'connect to the secondary')

        ha_tools.set_maintenance(secondary, True)

        # Trigger a refresh in various ways
        self.assertRaises(AutoReconnect, self.c_find_one.test.test.find_one)
        self.assertRaises(AutoReconnect, self.c_count.test.test.count)

        # Wait for the immediate refresh to complete - we're not waiting for
        # the periodic refresh, which has been disabled
        time.sleep(1)

        self.assertFalse(self.c_find_one.secondaries)
        self.assertEqual(partition_node(primary), self.c_find_one.primary)

        self.assertFalse(self.c_count.secondaries)
        self.assertEqual(partition_node(primary), self.c_count.primary)
예제 #31
0
    def test_primary_stepdown(self):
        c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(c.secondaries)))
        primary = c.primary
        ha_tools.stepdown_primary()

        # Wait for new primary
        patience_seconds = 30
        for _ in xrange(patience_seconds):
            sleep(1)
            rs_state = c._MongoReplicaSetClient__rs_state
            if rs_state.writer and rs_state.writer != primary:
                # New primary stepped up
                new_primary = _partition_node(ha_tools.get_primary())
                self.assertEqual(new_primary, rs_state.writer)
                new_secondaries = partition_nodes(ha_tools.get_secondaries())
                self.assertEqual(set(new_secondaries), rs_state.secondaries)
                break
        else:
            self.fail(
                "No new primary after %s seconds. Old primary was %s, current"
                " is %s" % (patience_seconds, primary, ha_tools.get_primary()))
예제 #32
0
    def test_ship_of_theseus(self):
        c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        db = c.get_database(
            "pymongo_test",
            write_concern=WriteConcern(w=len(c.secondaries) + 1))
        db.test.insert_one({})
        find_one = db.test.find_one

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()

        new_hosts = []
        for i in range(3):
            new_hosts.append(ha_tools.add_member())

            # RS closes all connections after reconfig.
            for j in xrange(30):
                try:
                    if ha_tools.get_primary():
                        break
                except (ConnectionFailure, OperationFailure):
                    pass

                time.sleep(1)
            else:
                self.fail("Couldn't recover from reconfig")

        # Wait for new members to join.
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            time.sleep(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)
        time.sleep(5)

        wait_until(lambda: (ha_tools.get_primary()
                            and len(ha_tools.get_secondaries()) == 2),
                   "fail over",
                   timeout=30)

        time.sleep(2 * self.heartbeat_frequency)

        # No error.
        find_one()
        find_one(read_preference=SECONDARY)

        # All members down.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members(new_hosts)

        # Should be able to reconnect to set even though original seed
        # list is useless. Use SECONDARY so we don't have to wait for
        # the election, merely for the client to detect members are up.
        time.sleep(2 * self.heartbeat_frequency)
        find_one(read_preference=SECONDARY)

        # Kill new members and switch back to original two members.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members([primary, secondary1])

        # Wait for members to figure out they're secondaries.
        wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
                   "detect two secondaries",
                   timeout=30)

        # Should be able to reconnect to set again.
        time.sleep(2 * self.heartbeat_frequency)
        find_one(read_preference=SECONDARY)
예제 #33
0
    def test_ship_of_theseus(self):
        c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        db = c.get_database(
            "pymongo_test",
            write_concern=WriteConcern(w=len(c.secondaries) + 1))
        db.test.insert_one({})
        find_one = db.test.find_one

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()

        new_hosts = []
        for i in range(3):
            new_hosts.append(ha_tools.add_member())

            # RS closes all connections after reconfig.
            for j in xrange(30):
                try:
                    if ha_tools.get_primary():
                        break
                except (ConnectionFailure, OperationFailure):
                    pass

                time.sleep(1)
            else:
                self.fail("Couldn't recover from reconfig")

        # Wait for new members to join.
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            time.sleep(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)
        time.sleep(5)

        wait_until(lambda: (ha_tools.get_primary()
                            and len(ha_tools.get_secondaries()) == 2),
                   "fail over",
                   timeout=30)

        time.sleep(2 * self.heartbeat_frequency)

        # No error.
        find_one()
        find_one(read_preference=SECONDARY)

        # All members down.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members(new_hosts)

        # Should be able to reconnect to set even though original seed
        # list is useless. Use SECONDARY so we don't have to wait for
        # the election, merely for the client to detect members are up.
        time.sleep(2 * self.heartbeat_frequency)
        find_one(read_preference=SECONDARY)

        # Kill new members and switch back to original two members.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members([primary, secondary1])

        # Wait for members to figure out they're secondaries.
        wait_until(lambda: len(ha_tools.get_secondaries()) == 2,
                   "detect two secondaries",
                   timeout=30)

        # Should be able to reconnect to set again.
        time.sleep(2 * self.heartbeat_frequency)
        find_one(read_preference=SECONDARY)
예제 #34
0
    def test_secondary_connection(self):
        self.c = yield motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name).open()

        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.motor_test
        yield db.test.remove({}, w=len(self.c.secondaries))

        # Wait for replication...
        w = len(self.c.secondaries) + 1
        yield db.test.insert({'foo': 'bar'}, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # A connection succeeds no matter the read preference
        for kwargs in [
            {
                'read_preference': PRIMARY
            },
            {
                'read_preference': PRIMARY_PREFERRED
            },
            {
                'read_preference': SECONDARY
            },
            {
                'read_preference': SECONDARY_PREFERRED
            },
            {
                'read_preference': NEAREST
            },
        ]:
            client = yield motor.MotorClient(primary_host, primary_port,
                                             **kwargs).open()

            self.assertEqual(primary_host, client.host)
            self.assertEqual(primary_port, client.port)
            self.assertTrue(client.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue((yield client.motor_test.test.find_one()))

            client = yield motor.MotorClient(secondary_host, secondary_port,
                                             **kwargs).open()
            self.assertEqual(secondary_host, client.host)
            self.assertEqual(secondary_port, client.port)
            self.assertFalse(client.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != PRIMARY:
                self.assertTrue((yield client.motor_test.test.find_one()))
            else:
                with assert_raises(AutoReconnect):
                    yield client.motor_test.test.find_one()

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), MotorClient
            # should do the same for unacknowledged writes.
            try:
                yield client.motor_test.test.insert({}, w=0)
            except AutoReconnect, e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into secondary client %s should'
                    'have raised exception' % client)

            # Test direct connection to an arbiter
            client = yield motor.MotorClient(arbiter_host, arbiter_port,
                                             **kwargs).open()

            self.assertEqual(arbiter_host, client.host)
            self.assertEqual(arbiter_port, client.port)
            self.assertFalse(client.is_primary)

            # See explanation above
            try:
                yield client.motor_test.test.insert({}, w=0)
            except AutoReconnect, e:
                self.assertEqual('not master', e.message)
예제 #35
0
    def test_secondary_connection(self):
        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        wait_until(lambda: len(self.c.secondaries), "discover secondary")
        # Wait for replication...
        w = len(self.c.secondaries) + 1
        db = self.c.get_database("pymongo_test",
                                 write_concern=WriteConcern(w=w))

        db.test.delete_many({})
        db.test.insert_one({'foo': 'bar'})

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # MongoClient succeeds no matter the read preference
        for kwargs in [
            {'read_preference': PRIMARY},
            {'read_preference': PRIMARY_PREFERRED},
            {'read_preference': SECONDARY},
            {'read_preference': SECONDARY_PREFERRED},
            {'read_preference': NEAREST},
        ]:
            client = MongoClient(
                primary_host,
                primary_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: primary_host == client.host,
                       "connect to primary")

            self.assertEqual(primary_port, client.port)
            self.assertTrue(client.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue(client.pymongo_test.test.find_one())

            client = MongoClient(
                secondary_host,
                secondary_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: secondary_host == client.host,
                       "connect to secondary")

            self.assertEqual(secondary_port, client.port)
            self.assertFalse(client.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != PRIMARY:
                self.assertTrue(client.pymongo_test.test.find_one())
            else:
                self.assertRaises(
                    AutoReconnect, client.pymongo_test.test.find_one)

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), MongoClient
            # should do the same for unacknowledged writes.
            try:
                client.get_database(
                    "pymongo_test",
                    write_concern=WriteConcern(w=0)).test.insert_one({})
            except AutoReconnect as e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into secondary client %s should'
                    'have raised exception' % (client,))

            # Test direct connection to an arbiter
            client = MongoClient(
                arbiter_host,
                arbiter_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: arbiter_host == client.host,
                       "connect to arbiter")

            self.assertEqual(arbiter_port, client.port)
            self.assertFalse(client.is_primary)
            
            # See explanation above
            try:
                client.get_database(
                    "pymongo_test",
                    write_concern=WriteConcern(w=0)).test.insert_one({})
            except AutoReconnect as e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into arbiter client %s should'
                    'have raised exception' % (client,))
예제 #36
0
    def test_secondary_connection(self):
        self.c = yield motor.MotorReplicaSetClient(
            self.seed, replicaSet=self.name).open()

        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.motor_test
        yield db.test.remove({}, w=len(self.c.secondaries))

        # Wait for replication...
        w = len(self.c.secondaries) + 1
        yield db.test.insert({'foo': 'bar'}, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # A connection succeeds no matter the read preference
        for kwargs in [
            {'read_preference': PRIMARY},
            {'read_preference': PRIMARY_PREFERRED},
            {'read_preference': SECONDARY},
            {'read_preference': SECONDARY_PREFERRED},
            {'read_preference': NEAREST},
        ]:
            client = yield motor.MotorClient(
                primary_host, primary_port, **kwargs).open()

            self.assertEqual(primary_host, client.host)
            self.assertEqual(primary_port, client.port)
            self.assertTrue(client.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue((yield client.motor_test.test.find_one()))

            client = yield motor.MotorClient(
                secondary_host, secondary_port, **kwargs).open()
            self.assertEqual(secondary_host, client.host)
            self.assertEqual(secondary_port, client.port)
            self.assertFalse(client.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != PRIMARY:
                self.assertTrue((
                    yield client.motor_test.test.find_one()))
            else:
                with assert_raises(AutoReconnect):
                    yield client.motor_test.test.find_one()

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), MotorClient
            # should do the same for unacknowledged writes.
            try:
                yield client.motor_test.test.insert({}, w=0)
            except AutoReconnect, e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into secondary client %s should'
                    'have raised exception' % client)

            # Test direct connection to an arbiter
            client = yield motor.MotorClient(
                arbiter_host, arbiter_port, **kwargs).open()

            self.assertEqual(arbiter_host, client.host)
            self.assertEqual(arbiter_port, client.port)
            self.assertFalse(client.is_primary)

            # See explanation above
            try:
                yield client.motor_test.test.insert({}, w=0)
            except AutoReconnect, e:
                self.assertEqual('not master', e.message)
예제 #37
0
    def test_ship_of_theseus(self):
        c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        db = c.pymongo_test
        db.test.insert({}, w=len(c.secondaries) + 1)
        find_one = db.test.find_one

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()

        new_hosts = [ha_tools.add_member() for _ in range(3)]

        # Wait for new members to join.
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            sleep(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)

        # Wait for primary.
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            sleep(1)
        else:
            self.fail("No failover")

        sleep(2 * MONITOR_INTERVAL)

        # No error.
        find_one()
        find_one(read_preference=SECONDARY)

        # All members down.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members(new_hosts)

        # Should be able to reconnect to set even though original seed
        # list is useless. Use SECONDARY so we don't have to wait for
        # the election, merely for the client to detect members are up.
        sleep(2 * MONITOR_INTERVAL)
        find_one(read_preference=SECONDARY)

        # Kill new members and switch back to original two members.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members([primary, secondary1])

        # Should be able to reconnect to set again.
        sleep(2 * MONITOR_INTERVAL)
        find_one(read_preference=SECONDARY)
예제 #38
0
    def test_secondary_connection(self):
        self.c = MongoClient(
            self.seed,
            replicaSet=self.name,
            serverSelectionTimeoutMS=self.server_selection_timeout)
        wait_until(lambda: len(self.c.secondaries), "discover secondary")
        # Wait for replication...
        w = len(self.c.secondaries) + 1
        db = self.c.get_database("pymongo_test",
                                 write_concern=WriteConcern(w=w))

        db.test.delete_many({})
        db.test.insert_one({'foo': 'bar'})

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # MongoClient succeeds no matter the read preference
        for kwargs in [
            {'read_preference': PRIMARY},
            {'read_preference': PRIMARY_PREFERRED},
            {'read_preference': SECONDARY},
            {'read_preference': SECONDARY_PREFERRED},
            {'read_preference': NEAREST},
        ]:
            client = MongoClient(
                primary_host,
                primary_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: primary_host == client.host,
                       "connect to primary")

            self.assertEqual(primary_port, client.port)
            self.assertTrue(client.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue(client.pymongo_test.test.find_one())

            client = MongoClient(
                secondary_host,
                secondary_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: secondary_host == client.host,
                       "connect to secondary")

            self.assertEqual(secondary_port, client.port)
            self.assertFalse(client.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != PRIMARY:
                self.assertTrue(client.pymongo_test.test.find_one())
            else:
                self.assertRaises(
                    AutoReconnect, client.pymongo_test.test.find_one)

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), MongoClient
            # should do the same for unacknowledged writes.
            try:
                client.get_database(
                    "pymongo_test",
                    write_concern=WriteConcern(w=0)).test.insert_one({})
            except AutoReconnect as e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into secondary client %s should'
                    'have raised exception' % (client,))

            # Test direct connection to an arbiter
            client = MongoClient(
                arbiter_host,
                arbiter_port,
                serverSelectionTimeoutMS=self.server_selection_timeout,
                **kwargs)
            wait_until(lambda: arbiter_host == client.host,
                       "connect to arbiter")

            self.assertEqual(arbiter_port, client.port)
            self.assertFalse(client.is_primary)

            # See explanation above
            try:
                client.get_database(
                    "pymongo_test",
                    write_concern=WriteConcern(w=0)).test.insert_one({})
            except AutoReconnect as e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into arbiter client %s should'
                    'have raised exception' % (client,))
예제 #39
0
    def test_secondary_connection(self):
        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.pymongo_test
        db.test.remove({}, safe=True, w=len(self.c.secondaries))

        # Wait for replication...
        w = len(self.c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # A Connection succeeds no matter the read preference
        for kwargs in [
            {'read_preference': PRIMARY},
            {'read_preference': PRIMARY_PREFERRED},
            {'read_preference': SECONDARY},
            {'read_preference': SECONDARY_PREFERRED},
            {'read_preference': NEAREST},
            {'slave_okay': True}
        ]:
            conn = Connection(primary_host,
                              primary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(primary_host, conn.host)
            self.assertEqual(primary_port, conn.port)
            self.assertTrue(conn.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue(conn.pymongo_test.test.find_one())

            conn = Connection(secondary_host,
                              secondary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(secondary_host, conn.host)
            self.assertEqual(secondary_port, conn.port)
            self.assertFalse(conn.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != PRIMARY:
                self.assertTrue(conn.pymongo_test.test.find_one())
            else:
                self.assertRaises(
                    AutoReconnect, conn.pymongo_test.test.find_one)

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), Connection
            # should do the same for unacknowledged writes.
            try:
                conn.pymongo_test.test.insert({}, safe=False)
            except AutoReconnect, e:
                self.assertEqual('not master', e.args[0])
            else:
                self.fail(
                    'Unacknowledged insert into secondary connection %s should'
                    'have raised exception' % conn)

            # Test direct connection to an arbiter
            conn = Connection(arbiter_host, arbiter_port, **kwargs)
            self.assertEqual(arbiter_host, conn.host)
            self.assertEqual(arbiter_port, conn.port)
            self.assertFalse(conn.is_primary)
            
            # See explanation above
            try:
                conn.pymongo_test.test.insert({}, safe=False)
            except AutoReconnect, e:
                self.assertEqual('not master', e.args[0])
예제 #40
0
    def test_ship_of_theseus(self):
        c = MongoReplicaSetClient(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)

        db = c.pymongo_test
        db.test.insert({}, w=len(c.secondaries) + 1)
        find_one = db.test.find_one

        primary = ha_tools.get_primary()
        secondary1 = ha_tools.get_random_secondary()

        new_hosts = []
        for i in range(3):
            new_hosts.append(ha_tools.add_member())

            # RS closes all connections after reconfig.
            for j in xrange(30):
                try:
                    if ha_tools.get_primary():
                        break
                except (ConnectionFailure, OperationFailure):
                    pass

                sleep(1)
            else:
                self.fail("Couldn't recover from reconfig")

        # Wait for new members to join.
        for _ in xrange(120):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 4:
                break

            sleep(1)
        else:
            self.fail("New secondaries didn't join")

        ha_tools.kill_members([primary, secondary1], 9)
        sleep(5)

        # Wait for primary.
        for _ in xrange(30):
            if ha_tools.get_primary() and len(ha_tools.get_secondaries()) == 2:
                break

            sleep(1)
        else:
            self.fail("No failover")

        sleep(2 * MONITOR_INTERVAL)

        # No error.
        find_one()
        find_one(read_preference=SECONDARY)

        # All members down.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members(new_hosts)

        # Should be able to reconnect to set even though original seed
        # list is useless. Use SECONDARY so we don't have to wait for
        # the election, merely for the client to detect members are up.
        sleep(2 * MONITOR_INTERVAL)
        find_one(read_preference=SECONDARY)

        # Kill new members and switch back to original two members.
        ha_tools.kill_members(new_hosts, 9)
        self.assertRaises(
            ConnectionFailure,
            find_one, read_preference=SECONDARY)

        ha_tools.restart_members([primary, secondary1])

        # Wait for members to figure out they're secondaries.
        for _ in xrange(30):
            try:
                if len(ha_tools.get_secondaries()) == 2:
                    break
            except ConnectionFailure:
                pass

            sleep(1)
        else:
            self.fail("Original members didn't become secondaries")

        # Should be able to reconnect to set again.
        sleep(2 * MONITOR_INTERVAL)
        find_one(read_preference=SECONDARY)
예제 #41
0
    def test_secondary_connection(self):
        self.c = ReplicaSetConnection(
            self.seed, replicaSet=self.name, use_greenlets=use_greenlets)
        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.pymongo_test
        db.test.remove({}, safe=True, w=len(self.c.secondaries))

        # Wait for replication...
        w = len(self.c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)
        arbiter_host, arbiter_port = ha_tools.get_arbiters()[0].split(':')
        arbiter_port = int(arbiter_port)

        # A Connection succeeds no matter the read preference
        for kwargs in [
            {'read_preference': ReadPreference.PRIMARY},
            {'read_preference': ReadPreference.PRIMARY_PREFERRED},
            {'read_preference': ReadPreference.SECONDARY},
            {'read_preference': ReadPreference.SECONDARY_PREFERRED},
            {'read_preference': ReadPreference.NEAREST},
            {'slave_okay': True}
        ]:
            conn = Connection(primary_host,
                              primary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(primary_host, conn.host)
            self.assertEqual(primary_port, conn.port)
            self.assertTrue(conn.is_primary)

            # Direct connection to primary can be queried with any read pref
            self.assertTrue(conn.pymongo_test.test.find_one())

            conn = Connection(secondary_host,
                              secondary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(secondary_host, conn.host)
            self.assertEqual(secondary_port, conn.port)
            self.assertFalse(conn.is_primary)

            # Direct connection to secondary can be queried with any read pref
            # but PRIMARY
            if kwargs.get('read_preference') != ReadPreference.PRIMARY:
                self.assertTrue(conn.pymongo_test.test.find_one())
            else:
                self.assertRaises(
                    AutoReconnect, conn.pymongo_test.test.find_one)

            # Since an attempt at an acknowledged write to a secondary from a
            # direct connection raises AutoReconnect('not master'), Connection
            # should do the same for unacknowledged writes.
            try:
                conn.pymongo_test.test.insert({}, safe=False)
            except AutoReconnect, e:
                self.assertEqual('not master', e.message)
            else:
                self.fail(
                    'Unacknowledged insert into secondary connection %s should'
                    'have raised exception' % conn)

            # Test direct connection to an arbiter
            conn = Connection(arbiter_host, arbiter_port, **kwargs)
            self.assertEqual(arbiter_host, conn.host)
            self.assertEqual(arbiter_port, conn.port)
            self.assertFalse(conn.is_primary)
            
            # See explanation above
            try:
                conn.pymongo_test.test.insert({}, safe=False)
            except AutoReconnect, e:
                self.assertEqual('not master', e.message)
예제 #42
0
    def test_secondary_connection(self):
        self.c = ReplicaSetConnection(self.seed,
                                      replicaSet=self.name,
                                      use_greenlets=use_greenlets)
        self.assertTrue(bool(len(self.c.secondaries)))
        db = self.c.pymongo_test
        db.test.remove({}, safe=True, w=len(self.c.secondaries))

        # Force replication...
        w = len(self.c.secondaries) + 1
        db.test.insert({'foo': 'bar'}, safe=True, w=w)

        # Test direct connection to a primary or secondary
        primary_host, primary_port = ha_tools.get_primary().split(':')
        primary_port = int(primary_port)
        (secondary_host,
         secondary_port) = ha_tools.get_secondaries()[0].split(':')
        secondary_port = int(secondary_port)

        self.assertTrue(
            Connection(primary_host, primary_port,
                       use_greenlets=use_greenlets).is_primary)

        self.assertTrue(
            Connection(
                primary_host,
                primary_port,
                use_greenlets=use_greenlets,
                read_preference=ReadPreference.PRIMARY_PREFERRED).is_primary)

        self.assertTrue(
            Connection(
                primary_host,
                primary_port,
                use_greenlets=use_greenlets,
                read_preference=ReadPreference.SECONDARY_PREFERRED).is_primary)

        self.assertTrue(
            Connection(primary_host,
                       primary_port,
                       use_greenlets=use_greenlets,
                       read_preference=ReadPreference.NEAREST).is_primary)

        self.assertTrue(
            Connection(primary_host,
                       primary_port,
                       use_greenlets=use_greenlets,
                       read_preference=ReadPreference.SECONDARY).is_primary)

        for kwargs in [
            {
                'read_preference': ReadPreference.PRIMARY_PREFERRED
            },
            {
                'read_preference': ReadPreference.SECONDARY
            },
            {
                'read_preference': ReadPreference.SECONDARY_PREFERRED
            },
            {
                'read_preference': ReadPreference.NEAREST
            },
            {
                'slave_okay': True
            },
        ]:
            conn = Connection(secondary_host,
                              secondary_port,
                              use_greenlets=use_greenlets,
                              **kwargs)
            self.assertEqual(secondary_host, conn.host)
            self.assertEqual(secondary_port, conn.port)
            self.assertFalse(conn.is_primary)
            self.assert_(conn.pymongo_test.test.find_one())

        # Test direct connection to an arbiter
        host, port = ha_tools.get_arbiters()[0].split(':')
        port = int(port)
        conn = Connection(host, port)
        self.assertEqual(host, conn.host)
        self.assertEqual(port, conn.port)
예제 #43
0
파일: main.py 프로젝트: trnl/mongo-rs-api
    def post(self, op):
        """ Respond to a POST
        """

        # Start RS
        if op == "start":
            request = self._parse_json(self.request.body)
            members = request["members"]

            try:
                # start RS and get its' params
                res = ha_tools.start_replica_set(members)
            except:
                raise tornado.httpserver._BadRequestException("Couldn't start RS!")

            rs_uri, rs_name = res
            rs_id = uuid.uuid4()

            self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_uri=rs_uri, rs_name=rs_name))

        # Stop rs
        elif op == "stop":
            try:
                ha_tools.kill_all_members()
            except:
                raise tornado.httpserver._BadRequestException("Couldn't stop RS!")

        # Get primary
        elif op == "get_primary":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_primary_uri = ha_tools.get_primary()

            self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_primary_uri=rs_primary_uri))

        # Get secondaries
        elif op == "get_secondaries":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_secondaries_uris = ha_tools.get_secondaries()

            self.write(
                self._template.load(op + self._ext).generate(rs_id=rs_id, rs_secondaries_uris=rs_secondaries_uris)
            )

        # Get arbiters
        elif op == "get_arbiters":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_arbiters_uris = ha_tools.get_arbiters()

            self.write(self._template.load(op + self._ext).generate(rs_id=rs_id, rs_arbiters_uris=rs_arbiters_uris))

        # Kill primary
        elif op == "kill_primary":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_killed_primary_uri = ha_tools.kill_primary()

            self.write(
                self._template.load(op + self._ext).generate(rs_id=rs_id, rs_killed_primary_uri=rs_killed_primary_uri)
            )

        # Kill secondary
        elif op == "kill_secondary":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_killed_secondary_uri = ha_tools.kill_secondary()

            self.write(
                self._template.load(op + self._ext).generate(
                    rs_id=rs_id, rs_killed_secondary_uri=rs_killed_secondary_uri
                )
            )

        # Kill all secondaries
        elif op == "kill_all_secondaries":
            request = self._parse_json(self.request.body)
            rs_id = request["rs"]["id"]

            rs_killed_secondaries_uris = ha_tools.kill_all_secondaries()

            self.write(
                self._template.load(op + self._ext).generate(
                    rs_id=rs_id, rs_killed_secondaries_uris=rs_killed_secondaries_uris
                )
            )