Ejemplo n.º 1
0
    def test_7_remaining_hbc_connection( self ):
        util.print_frame()

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
            util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )

            # check smr
            success = False
            expected = 0
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
            util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
            util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
Ejemplo n.º 2
0
    def test_7_remaining_hbc_connection( self ):
        util.print_frame()

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
            util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )

            # check smr
            success = False
            expected = 0
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
            util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
            util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
Ejemplo n.º 3
0
    def test_3_heartbeat_target_connection_count(self):
        util.print_frame()

        util.log('wait until all connections are established')
        for i in range(1, 8):
            time.sleep(1)
            util.log('%d sec' % i)

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(
                server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(
                server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'],
                                              server['id'])
            ret = util.cm_command(self.leader_cm['ip'],
                                  self.leader_cm['cm_port'], cmd)
            jobj = json.loads(ret)
            self.assertEqual(jobj['state'], 'success',
                             'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
            util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))

            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(
                    server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals(
                success, True,
                'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d'
                % (server['id'], server['ip'], server['redis_port'], after_cnt,
                   before_cnt_redis))
            util.log(
                'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
                % (server['id'], server['ip'], server['redis_port'], after_cnt,
                   before_cnt_redis))

            # check smr
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(
                    server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals(
                success, True,
                'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
                % (server['id'], server['ip'], server['smr_mgmt_port'],
                   after_cnt, expected, before_cnt_smr))
            util.log(
                'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
                % (server['id'], server['ip'], server['smr_mgmt_port'],
                   after_cnt, expected, before_cnt_smr))

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'],
                                                      server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            ret = util.cm_command(self.leader_cm['ip'],
                                  self.leader_cm['cm_port'], cmd)
            jobj = json.loads(ret)
            self.assertEqual(jobj['state'], 'success',
                             'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]))
            util.log('succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]))

            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(
                    server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals(
                success, True,
                'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
                % (server['id'], server['ip'], server['gateway_port'],
                   after_cnt, expected))
            util.log(
                'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
                % (server['id'], server['ip'], server['gateway_port'],
                   after_cnt, expected))
Ejemplo n.º 4
0
    def test_3_heartbeat_target_connection_count( self ):
        util.print_frame()

        util.log( 'wait until all connections are established' )
        for i in range(1, 8):
            time.sleep(1)
            util.log( '%d sec' % i )

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced' % (self.cluster['cluster_name'], server['id'])
            ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
            jobj = json.loads(ret)
            self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
            util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )

            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected:n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
            util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )

            # check smr
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
            util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            ret = util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)
            jobj = json.loads(ret)
            self.assertEqual( jobj['state'], 'success', 'failed : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )
            util.log( 'succeeded : cmd="%s", reply="%s"' % (cmd, ret[:-2]) )

            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
            util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
Ejemplo n.º 5
0
    def test_7_remaining_hbc_connection(self):
        util.print_frame()

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(
                server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(
                server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (
                self.cluster['cluster_name'], server['id'],
                self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'],
                            cmd)

        for server in self.cluster['servers']:
            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(
                    server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals(
                success, True,
                'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
                % (server['id'], server['ip'], server['redis_port'], after_cnt,
                   before_cnt_redis))
            util.log(
                'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d'
                % (server['id'], server['ip'], server['redis_port'], after_cnt,
                   before_cnt_redis))

            # check smr
            success = False
            expected = 0
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(
                    server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals(
                success, True,
                'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
                % (server['id'], server['ip'], server['smr_mgmt_port'],
                   after_cnt, expected, before_cnt_smr))
            util.log(
                'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d'
                % (server['id'], server['ip'], server['smr_mgmt_port'],
                   after_cnt, expected, before_cnt_smr))

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'],
                                                      server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'],
                            cmd)

        for server in self.cluster['servers']:
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(
                    server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals(
                success, True,
                'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
                % (server['id'], server['ip'], server['gateway_port'],
                   after_cnt, expected))
            util.log(
                'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.'
                % (server['id'], server['ip'], server['gateway_port'],
                   after_cnt, expected))

        # Go back to initial configuration
        # Cleanup PG
        self.assertTrue(
            util.cm_success(
                util.cm_command(
                    self.leader_cm['ip'], self.leader_cm['cm_port'],
                    'pg_del %s %d' %
                    (self.cluster['cluster_name'],
                     self.cluster['servers'][0]['pg_id'])))[0])

        # Cleanup processes of PGS and GW
        for s in self.cluster['servers']:
            self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']),
                             'failed to kill redis %d process' % s['id'])
            self.assertEqual(
                0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']),
                'failed to kill smr %d process' % s['id'])
            self.assertEqual(0,
                             util.shutdown_gateway(s['id'], s['gateway_port']),
                             'failed to kill gw %d process' % s['id'])

        # Recover PG
        self.assertTrue(
            util.install_pg(self.cluster,
                            self.cluster['servers'],
                            self.cluster['servers'][0],
                            start_gw=True),
            'failed to recover PGS and GW in a PM')
    def test_7_remaining_hbc_connection( self ):
        util.print_frame()

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
            util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )

            # check smr
            success = False
            expected = 0
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
            util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
            util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )

        # Go back to initial configuration
        # Cleanup PG
        self.assertTrue(util.cm_success(util.cm_command(
            self.leader_cm['ip'], self.leader_cm['cm_port'], 
            'pg_del %s %d' % (self.cluster['cluster_name'], self.cluster['servers'][0]['pg_id'])))[0])

        # Cleanup processes of PGS and GW
        for s in self.cluster['servers']:
            self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']), 
                'failed to kill redis %d process' % s['id'])
            self.assertEqual(0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']), 
                'failed to kill smr %d process' % s['id'])
            self.assertEqual(0, util.shutdown_gateway(s['id'], s['gateway_port']),
                'failed to kill gw %d process' % s['id'])

        # Recover PG
        self.assertTrue(
                util.install_pg(self.cluster, self.cluster['servers'], self.cluster['servers'][0], start_gw=True),
                'failed to recover PGS and GW in a PM')