コード例 #1
0
    def test_rebalance_leaders_unbalanced_case3(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Imbalanced 0 and 2. No re-balance possible.
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0']),
                ((u'T2', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify still leader-imbalanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(ct.brokers.values()),
        )
        assert leader_imbal == 1
        # No change in assignment
        assert sorted(ct.assignment) == sorted(assignment)
コード例 #2
0
    def test_revoke_leadership_single_broker(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        assignment = dict([
            ((u'T0', 0), ['2', '0']),
            ((u'T1', 0), ['2', '1']),
            ((u'T1', 1), ['0', '2']),
            ((u'T2', 0), ['1', '0']),
        ])
        ct = create_cluster_topology(assignment, broker_range(3))
        cb = create_balancer(ct)
        cb.revoke_leadership(['2'])

        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in six.itervalues(ct.brokers)
        }
        _, total_movements = \
            calculate_partition_movement(assignment, ct.assignment)
        # Get net imbalance statistics excluding brokers to be revoked
        # leadership from
        brokers = [
            b for b in six.itervalues(ct.brokers) if b.id not in ['2', '3']
        ]
        new_net_imbal = get_net_imbalance(get_broker_leader_counts(brokers))
        # Verify that broker '2' is not leader of any partition
        assert new_leaders_per_broker['2'] == 0
        assert new_leaders_per_broker['1'] == 2
        assert new_leaders_per_broker['0'] == 2
        # Assert no partition movements
        assert total_movements == 0
        # Assert remaining brokers are balanced with leader count
        assert new_net_imbal == 0
コード例 #3
0
    def test_rebalance_leaders_unbalanced_case2(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # (Broker: leader-count): {0: 2, 1: 1, 2:0}
        # opt-count: 3/3 = 1, extra-count = 0
        # Leader-imbalance-value: 1
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T1', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(ct.brokers.values()),
        )
        assert leader_imbal == 0
コード例 #4
0
    def test_rebalance_leaders_balanced_case1(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        # Already balanced-assignment with evenly-distributed
        # (broker-id: leader-count): {0: 1, 1:1, 2:1}
        # opt-count: 3/3 = 1, extra-count: 3%3 = 0
        assignment = dict([
            ((u'T0', 0), ['1', '2']),
            ((u'T0', 1), ['2', '0']),
            ((u'T1', 0), ['0', '2']),
        ])
        ct = create_cluster_topology(assignment, broker_range(3))
        orig_assignment = ct.assignment

        cb = create_balancer(ct)
        cb.rebalance_leaders()
        net_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())), )

        # No changed in already-balanced assignment
        assert orig_assignment == ct.assignment
        # Assert leader-balanced
        assert net_imbal == 0
コード例 #5
0
    def test_rebalance_leaders_unbalanced_case5(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Special case, wherein none under-balanced
        # but 0 is overbalanced
        assignment = dict(
            [
                ((u'T1', 1), ['0', '1']),
                ((u'T2', 0), ['0']),
                ((u'T2', 1), ['0']),
                ((u'T3', 0), ['2', '3']),
                ((u'T3', 1), ['3', '1']),
                ((u'T4', 0), ['1']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(ct.brokers.values()),
        )
        assert leader_imbal == 0
コード例 #6
0
    def test_rebalance_leaders_unbalanced_case1(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        # Balance leader-imbalance successfully
        # (broker-id: leader-count): {0: 0, 1:2, 2:1}
        # Net-leader-imbalance: 1
        # opt-count: 3/3 = 1, extra-count: 3%3 = 0
        assignment = dict([
            ((u'T0', 0), ['1', '2']),
            ((u'T0', 1), ['2', '0']),
            ((u'T1', 0), ['1', '0']),
        ])
        ct = create_cluster_topology(assignment, broker_range(3))
        orig_assignment = ct.assignment

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify if valid-leader assignment
        self.assert_leader_valid(orig_assignment, ct.assignment)
        # New-leader imbalance-count be less than previous imbal count
        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in six.itervalues(ct.brokers)
        }
        new_leader_imbal = get_net_imbalance(
            list(new_leaders_per_broker.values()))
        # Verify leader-balanced
        assert new_leader_imbal == 0
        # Verify partitions-changed assignment
        assert new_leaders_per_broker['0'] == 1
        assert new_leaders_per_broker['1'] == 1
        assert new_leaders_per_broker['2'] == 1
コード例 #7
0
    def test_rebalance_leaders_unbalanced_case3(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Imbalanced 0 and 2. No re-balance possible.
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0']),
                ((u'T2', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify still leader-imbalanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 1
        # No change in assignment
        assert sorted(ct.assignment) == sorted(assignment)
コード例 #8
0
    def test_rebalance_leaders_unbalanced_case2d(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Broker-2 imbalanced with same brokers
        # Broker-2 requests leadership from same broker-1 twice
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T1', 2), ['0']),
                ((u'T1', 3), ['1', '2']),
                ((u'T1', 4), ['0', '1']),
                ((u'T1', 5), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(ct.brokers.values()),
        )
        assert leader_imbal == 0
コード例 #9
0
    def test_rebalance_leaders_unbalanced_case2d(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Broker-2 imbalanced with same brokers
        # Broker-2 requests leadership from same broker-1 twice
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T1', 2), ['0']),
                ((u'T1', 3), ['1', '2']),
                ((u'T1', 4), ['0', '1']),
                ((u'T1', 5), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #10
0
    def test_rebalance_leaders_unbalanced_case2e(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Imbalance-val 2
        # Multiple imbalanced brokers (2, 5) gets non-follower balanced
        # from multiple brokers (1,4)
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T2', 0), ['0']),
                ((u'T3', 0), ['4', '5']),
                ((u'T3', 1), ['3', '4']),
                ((u'T4', 0), ['3']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(6))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #11
0
    def test_rebalance_leaders_unbalanced_case2a(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        # (Broker: leader-count): {0: 2, 1: 1, 2:0, 3:1}
        # opt-count: 3/4 = 1, extra-count = 3
        # Leader-imbalance-value: 1
        # imbalanced-broker: 0,2; balanced-brokers: 1,3
        assignment = dict([
            ((u'T0', 0), ['3', '2']),
            ((u'T0', 1), ['1', '3']),
            ((u'T1', 1), ['0', '1']),
            ((u'T1', 0), ['0']),
        ])
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())), )
        assert leader_imbal == 0
        # Verify that (T0, 1) also swapped even if 1 and 3 were balanced
        # Rebalancing through non-followers
        replica_ids = [b.id for b in ct.partitions[('T0', 1)].replicas]
        assert replica_ids == ['3', '1']
コード例 #12
0
    def test_rebalance_leaders_unbalanced_case2c(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Broker-2 imbalance value: 2 with different brokers
        # Broker-2 requests leadership from multiple brokers (0, 1) once
        assignment = dict(
            [
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T2', 0), ['0']),
                ((u'T2', 1), ['0']),
                ((u'T3', 0), ['3', '2']),
                ((u'T3', 1), ['1', '3']),
                ((u'T4', 0), ['1']),
                ((u'T4', 2), ['3']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #13
0
    def test_rebalance_leaders_unbalanced_case2(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # (Broker: leader-count): {0: 2, 1: 1, 2:0}
        # opt-count: 3/3 = 1, extra-count = 0
        # Leader-imbalance-value: 1
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T1', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #14
0
    def test_rebalance_leaders_unbalanced_case2a(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # (Broker: leader-count): {0: 2, 1: 1, 2:0, 3:1}
        # opt-count: 3/4 = 1, extra-count = 3
        # Leader-imbalance-value: 1
        # imbalanced-broker: 0,2; balanced-brokers: 1,3
        assignment = dict(
            [
                ((u'T0', 0), ['3', '2']),
                ((u'T0', 1), ['1', '3']),
                ((u'T1', 1), ['0', '1']),
                ((u'T1', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
        # Verify that (T0, 1) also swapped even if 1 and 3 were balanced
        # Rebalancing through non-followers
        replica_ids = [b.id for b in ct.partitions[('T0', 1)].replicas]
        assert replica_ids == ['3', '1']
コード例 #15
0
    def test_rebalance_leaders_unbalanced_case2c(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        # Broker-2 imbalance value: 2 with different brokers
        # Broker-2 requests leadership from multiple brokers (0, 1) once
        assignment = dict([
            ((u'T1', 0), ['1', '2']),
            ((u'T1', 1), ['0', '1']),
            ((u'T2', 0), ['0']),
            ((u'T2', 1), ['0']),
            ((u'T3', 0), ['3', '2']),
            ((u'T3', 1), ['1', '3']),
            ((u'T4', 0), ['1']),
            ((u'T4', 2), ['3']),
        ])
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())), )
        assert leader_imbal == 0
コード例 #16
0
    def test_rebalance_leaders_balanced_case2(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Already balanced-assignment NOT evenly-distributed
        # (broker-id: leader-count): {0: 1, 1:1, 2:1}
        # opt-count: 2/3 = 0, extra-count: 2%3 = 2
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T0', 1), ['2', '0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))
        orig_assignment = ct.assignment

        cb = create_balancer(ct)
        cb.rebalance_leaders()
        net_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )

        # No changed in already-balanced assignment
        assert orig_assignment == ct.assignment
        # Assert leader-balanced
        assert net_imbal == 0
コード例 #17
0
    def test_rebalance_leaders_unbalanced_case2e(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        # Imbalance-val 2
        # Multiple imbalanced brokers (2, 5) gets non-follower balanced
        # from multiple brokers (1,4)
        assignment = dict([
            ((u'T1', 0), ['1', '2']),
            ((u'T1', 1), ['0', '1']),
            ((u'T2', 0), ['0']),
            ((u'T3', 0), ['4', '5']),
            ((u'T3', 1), ['3', '4']),
            ((u'T4', 0), ['3']),
        ])
        ct = create_cluster_topology(assignment, broker_range(6))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())), )
        assert leader_imbal == 0
コード例 #18
0
    def test_rebalance_leaders_unbalanced_case5(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Special case, wherein none under-balanced
        # but 0 is overbalanced
        assignment = dict(
            [
                ((u'T1', 1), ['0', '1']),
                ((u'T2', 0), ['0']),
                ((u'T2', 1), ['0']),
                ((u'T3', 0), ['2', '3']),
                ((u'T3', 1), ['3', '1']),
                ((u'T4', 0), ['1']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify leader-balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #19
0
def display_leader_imbalance(cluster_topologies):
    """Display leader count and weight imbalance statistics.

    :param cluster_topologies: A dictionary mapping a string name to a
        ClusterTopology object.
    """
    broker_ids = list(next(six.itervalues(cluster_topologies)).brokers.keys())
    assert all(
        set(broker_ids) == set(cluster_topology.brokers.keys())
        for cluster_topology in six.itervalues(cluster_topologies))

    broker_leader_counts = [
        stats.get_broker_leader_counts(cluster_topology.brokers[broker_id]
                                       for broker_id in broker_ids)
        for cluster_topology in six.itervalues(cluster_topologies)
    ]
    broker_leader_weights = [
        stats.get_broker_leader_weights(cluster_topology.brokers[broker_id]
                                        for broker_id in broker_ids)
        for cluster_topology in six.itervalues(cluster_topologies)
    ]

    _display_table_title_multicolumn(
        'Leader Count',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        broker_leader_counts,
    )

    print('')

    _display_table_title_multicolumn(
        'Leader weight',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        broker_leader_weights,
    )

    for name, blc, blw in zip(list(cluster_topologies.keys()),
                              broker_leader_counts, broker_leader_weights):
        print('\n'
              '{name}'
              'Leader count imbalance: {net_imbalance}\n'
              'Broker leader weight mean: {weight_mean}\n'
              'Broker leader weight stdev: {weight_stdev}\n'
              'Broker leader weight cv: {weight_cv}'.format(
                  name='' if len(cluster_topologies) == 1 else name + '\n',
                  net_imbalance=stats.get_net_imbalance(blc),
                  weight_mean=stats.mean(blw),
                  weight_stdev=stats.standard_deviation(blw),
                  weight_cv=stats.coefficient_of_variation(blw),
              ))
コード例 #20
0
    def test_rebalance_leaders_unbalanced_case4(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Imbalanced assignment
        # Partial leader-imbalance possible
        # (Broker: leader-count): {0: 3, 1: 1, 2:0}
        # opt-count: 5/3 = 1, extra-count = 2
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T0', 1), ['0', '2']),
                ((u'T1', 0), ['0']),
                ((u'T1', 1), ['0']),
                ((u'T1', 2), ['0']),
            ]
        )

        ct = create_cluster_topology(assignment, broker_range(3))
        net_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in six.itervalues(ct.brokers)
        }
        new_net_imbal = get_net_imbalance(list(new_leaders_per_broker.values()))
        # Verify that net-imbalance has reduced but not zero
        assert new_net_imbal > 0 and new_net_imbal < net_imbal
        # Verify the changes in leaders-per-broker count
        assert new_leaders_per_broker['2'] == 1
        assert new_leaders_per_broker['1'] == 1
        assert new_leaders_per_broker['0'] == 3
コード例 #21
0
    def test_rebalance_leaders_unbalanced_case4(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Imbalanced assignment
        # Partial leader-imbalance possible
        # (Broker: leader-count): {0: 3, 1: 1, 2:0}
        # opt-count: 5/3 = 1, extra-count = 2
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T0', 1), ['0', '2']),
                ((u'T1', 0), ['0']),
                ((u'T1', 1), ['0']),
                ((u'T1', 2), ['0']),
            ]
        )

        ct = create_cluster_topology(assignment, broker_range(3))
        net_imbal = get_net_imbalance(
            get_broker_leader_counts(ct.brokers.values()),
        )

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in ct.brokers.itervalues()
        }
        new_net_imbal = get_net_imbalance(new_leaders_per_broker.values())
        # Verify that net-imbalance has reduced but not zero
        assert new_net_imbal > 0 and new_net_imbal < net_imbal
        # Verify the changes in leaders-per-broker count
        assert new_leaders_per_broker['2'] == 1
        assert new_leaders_per_broker['1'] == 1
        assert new_leaders_per_broker['0'] == 3
コード例 #22
0
    def test_rebalance_leaders_unbalanced_case2b(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        assignment = dict([
            ((u'T0', 0), ['3', '2']),
            ((u'T1', 0), ['1', '2']),
            ((u'T1', 1), ['0', '1']),
            ((u'T2', 0), ['0']),
        ])
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())), )
        assert leader_imbal == 0
コード例 #23
0
    def test_revoke_leadership_multiple_brokers(
        self,
        create_balancer,
        create_cluster_topology,
    ):
        assignment = dict(
            [
                ((u'T0', 0), ['2', '0']),
                ((u'T1', 0), ['2', '1']),
                ((u'T1', 1), ['0', '2']),
                ((u'T2', 0), ['0', '3']),
                ((u'T3', 0), ['3', '1']),
                ((u'T3', 1), ['3', '1']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))
        cb = create_balancer(ct)
        cb.revoke_leadership(['2', '3'])

        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in six.itervalues(ct.brokers)
        }
        _, total_movements = \
            calculate_partition_movement(assignment, ct.assignment)
        # Get net imbalance statistics excluding brokers to be revoked
        # leadership from
        brokers = [
            b for b in six.itervalues(ct.brokers)
            if b.id not in ['2', '3']
        ]
        new_net_imbal = get_net_imbalance(get_broker_leader_counts(brokers))
        # Verify that broker '2' and '3' is not leader of any partition
        assert new_leaders_per_broker['0'] == 3
        assert new_leaders_per_broker['1'] == 3
        assert new_leaders_per_broker['2'] == 0
        assert new_leaders_per_broker['3'] == 0
        # Assert no partition movements
        assert total_movements == 0
        # Assert remaining brokers are balanced with leader count
        assert new_net_imbal == 0
コード例 #24
0
    def test_rebalance_leaders_unbalanced_case2b(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        assignment = dict(
            [
                ((u'T0', 0), ['3', '2']),
                ((u'T1', 0), ['1', '2']),
                ((u'T1', 1), ['0', '1']),
                ((u'T2', 0), ['0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(4))

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify balanced
        leader_imbal = get_net_imbalance(
            get_broker_leader_counts(list(ct.brokers.values())),
        )
        assert leader_imbal == 0
コード例 #25
0
    def test_rebalance_leaders_unbalanced_case1(
            self,
            create_balancer,
            create_cluster_topology,
    ):
        # Balance leader-imbalance successfully
        # (broker-id: leader-count): {0: 0, 1:2, 2:1}
        # Net-leader-imbalance: 1
        # opt-count: 3/3 = 1, extra-count: 3%3 = 0
        assignment = dict(
            [
                ((u'T0', 0), ['1', '2']),
                ((u'T0', 1), ['2', '0']),
                ((u'T1', 0), ['1', '0']),
            ]
        )
        ct = create_cluster_topology(assignment, broker_range(3))
        orig_assignment = ct.assignment

        cb = create_balancer(ct)
        cb.rebalance_leaders()

        # Verify if valid-leader assignment
        self.assert_leader_valid(orig_assignment, ct.assignment)
        # New-leader imbalance-count be less than previous imbal count
        new_leaders_per_broker = {
            broker.id: broker.count_preferred_replica()
            for broker in six.itervalues(ct.brokers)
        }
        new_leader_imbal = get_net_imbalance(list(new_leaders_per_broker.values()))
        # Verify leader-balanced
        assert new_leader_imbal == 0
        # Verify partitions-changed assignment
        assert new_leaders_per_broker['0'] == 1
        assert new_leaders_per_broker['1'] == 1
        assert new_leaders_per_broker['2'] == 1
コード例 #26
0
ファイル: display.py プロジェクト: Yelp/kafka-utils
def display_leader_imbalance(cluster_topologies):
    """Display leader count and weight imbalance statistics.

    :param cluster_topologies: A dictionary mapping a string name to a
        ClusterTopology object.
    """
    broker_ids = list(next(six.itervalues(cluster_topologies)).brokers.keys())
    assert all(
        set(broker_ids) == set(cluster_topology.brokers.keys())
        for cluster_topology in six.itervalues(cluster_topologies)
    )

    broker_leader_counts = [
        stats.get_broker_leader_counts(
            cluster_topology.brokers[broker_id]
            for broker_id in broker_ids
        )
        for cluster_topology in six.itervalues(cluster_topologies)
    ]
    broker_leader_weights = [
        stats.get_broker_leader_weights(
            cluster_topology.brokers[broker_id]
            for broker_id in broker_ids
        )
        for cluster_topology in six.itervalues(cluster_topologies)
    ]

    _display_table_title_multicolumn(
        'Leader Count',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        broker_leader_counts,
    )

    print('')

    _display_table_title_multicolumn(
        'Leader weight',
        'Brokers',
        broker_ids,
        list(cluster_topologies.keys()),
        broker_leader_weights,
    )

    for name, blc, blw in zip(
            list(cluster_topologies.keys()),
            broker_leader_counts,
            broker_leader_weights
    ):
        print(
            '\n'
            '{name}'
            'Leader count imbalance: {net_imbalance}\n'
            'Broker leader weight mean: {weight_mean}\n'
            'Broker leader weight stdev: {weight_stdev}\n'
            'Broker leader weight cv: {weight_cv}'
            .format(
                name='' if len(cluster_topologies) == 1 else name + '\n',
                net_imbalance=stats.get_net_imbalance(blc),
                weight_mean=stats.mean(blw),
                weight_stdev=stats.stdevp(blw),
                weight_cv=stats.coefficient_of_variation(blw),
            )
        )
コード例 #27
0
ファイル: stats_test.py プロジェクト: Yelp/kafka-utils
def test_get_net_imbalance_imbalanced_equal():
    assert stats.get_net_imbalance([3, 2, 3, 4, 3]) == 1
コード例 #28
0
ファイル: stats_test.py プロジェクト: Yelp/kafka-utils
def test_get_net_imbalance_imbalanced_unequal():
    assert stats.get_net_imbalance([3, 2, 4, 4, 4]) == 1
コード例 #29
0
def test_get_net_imbalance_imbalanced_unequal():
    assert stats.get_net_imbalance([3, 2, 4, 4, 4]) == 1
コード例 #30
0
def test_get_net_imbalance_imbalanced_equal():
    assert stats.get_net_imbalance([3, 2, 3, 4, 3]) == 1