예제 #1
0
    def test_gateway_lookup( self ):
        util.print_frame()

        # check initial states
        for server in self.cluster['servers']:
            success = False
            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('cZxid'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # shutdown gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
            self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )

            for try_cnt in range( 10 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('Node does not exist'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # restart gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
            self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )

            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('cZxid'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        return 0
예제 #2
0
    def test_gateway_lookup( self ):
        util.print_frame()

        # check initial states
        for server in self.cluster['servers']:
            success = False
            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd(cmd)
                if ret['exitcode'] == 'OK':
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # shutdown gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
            self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )

            for try_cnt in range( 10 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                if ret['exitcode'] == 'ZK_NO_NODE':
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # restart gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
            self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )

            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                if ret['exitcode'] == 'OK':
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        return 0
예제 #3
0
def cleanup_zookeeper_root():
    max_try = 20
    for i in range(max_try):
        print 'try cleanup zookeeper, i:%d' % i

        if util.zk_cmd('get /RC')['exitcode'] == 'ZK_NO_NODE':
            return 0

        ret = util.zk_cmd('rmr /RC')
        if ret['exitcode'] == 'OK':
            return 0
        else:
            util.log('failed to clean up zookeeper, ret:%s' % ret)
        time.sleep(0.2)
    return -1
예제 #4
0
def cleanup_zookeeper_root():
    max_try = 20
    for i in range( max_try ):
        print 'try cleanup zookeeper, i:%d' % i

        if util.zk_cmd( 'get /RC' )['exitcode'] == 'ZK_NO_NODE':
            return 0

        ret = util.zk_cmd( 'rmr /RC' )
        if ret['exitcode'] == 'OK':
            return 0
        else:
            util.log('failed to clean up zookeeper, ret:%s' % ret)
        time.sleep(0.2)
    return -1
예제 #5
0
def get_cluster_conf_from_zk(cluster_name):
    # Cluster
    cluster_conf = {
        'clusterName':
        cluster_name,
        'clusterData':
        json.loads(util.zk_cmd('get /RC/CLUSTER/%s' % cluster_name)['data'])
    }

    # PG
    cluster_conf['pgList'] = __znode_to_dict(
        'pgId',
        util.zk_get_children_with_data('/RC/CLUSTER/%s/PG' %
                                       cluster_conf['clusterName']))

    # PGS
    cluster_conf['pgsList'] = __znode_to_dict(
        'pgsId',
        util.zk_get_children_with_data('/RC/CLUSTER/%s/PGS' %
                                       cluster_conf['clusterName']))

    # GW
    cluster_conf['gwList'] = __znode_to_dict(
        'gwId',
        util.zk_get_children_with_data('/RC/CLUSTER/%s/GW' %
                                       cluster_conf['clusterName']))

    # GW lookup
    cluster_conf['gwLookup'] = __znode_to_dict(
        'gwId',
        util.zk_get_children_with_data('/RC/NOTIFICATION/CLUSTER/%s/GW' %
                                       cluster_conf['clusterName']))

    return cluster_conf
예제 #6
0
def check_gateway_affinity(zone_conf):
    for cluster_conf in zone_conf['cluster']:
        expected = get_gateway_affinity_cluster(cluster_conf)
        affinity = json.loads(util.zk_cmd('get /RC/NOTIFICATION/CLUSTER/%s/AFFINITY' % cluster_conf['clusterName'].encode('ascii'))['data'])
        for x, y in zip(expected, affinity):
            if len(compare_dict(x, y)) != 0:
                util.log('wrong gateway affinity of %s, %s != %s' % (cluster_conf['clusterName'], x, y))
                return False
    return True
예제 #7
0
def get_pm_conf_from_zk(pm_name):
    return {
        'pmName' : pm_name, 
        'pmInfo' : json.loads(util.zk_cmd('get /RC/PM/%s' % pm_name)['data']),
        'clusterList' : sorted(
            map(lambda x: {'name' : x['name'] , 'data' : sort_pm_data(json.loads(x['data']))}, 
                util.zk_get_children_with_data('/RC/PM/%s' % pm_name)), 
            key = lambda x: x['name'])
    }
예제 #8
0
def check_gateway_affinity(zone_conf):
    for cluster_conf in zone_conf['cluster']:
        expected = get_gateway_affinity_cluster(cluster_conf)
        affinity = json.loads(
            util.zk_cmd('get /RC/NOTIFICATION/CLUSTER/%s/AFFINITY' %
                        cluster_conf['clusterName'].encode('ascii'))['data'])
        for x, y in zip(expected, affinity):
            if len(compare_dict(x, y)) != 0:
                util.log('wrong gateway affinity of %s, %s != %s' %
                         (cluster_conf['clusterName'], x, y))
                return False
    return True
예제 #9
0
def get_pm_conf_from_zk(pm_name):
    return {
        'pmName':
        pm_name,
        'pmInfo':
        json.loads(util.zk_cmd('get /RC/PM/%s' % pm_name)['data']),
        'clusterList':
        sorted(map(
            lambda x: {
                'name': x['name'],
                'data': sort_pm_data(json.loads(x['data']))
            }, util.zk_get_children_with_data('/RC/PM/%s' % pm_name)),
               key=lambda x: x['name'])
    }
예제 #10
0
def cleanup_zookeeper_root():
    max_try = 20
    for i in range( max_try ):
        print 'try cleanup zookeeper, i:%d' % i
        ret = util.zk_cmd( 'rmr /RC' )
        err = ret['err']
        if err == '':
            return 0
        elif -1 != err.find( 'Node does not exist: /RC'):
            if err.count('/') == 1:
                return 0
        else:
            util.log('failed to clean up zookeeper, err:%s' % err)
        time.sleep(0.2)
    return -1
예제 #11
0
def get_cluster_conf_from_zk(cluster_name):
    # Cluster
    cluster_conf = {
        'clusterName' : cluster_name,
        'clusterData' : json.loads(util.zk_cmd('get /RC/CLUSTER/%s' % cluster_name)['data'])
    }

    # PG
    cluster_conf['pgList'] = __znode_to_dict(
            'pgId', util.zk_get_children_with_data('/RC/CLUSTER/%s/PG' % cluster_conf['clusterName']))

    # PGS
    cluster_conf['pgsList'] = __znode_to_dict(
            'pgsId', util.zk_get_children_with_data('/RC/CLUSTER/%s/PGS' % cluster_conf['clusterName'])) 

    # GW
    cluster_conf['gwList'] = __znode_to_dict(
            'gwId', util.zk_get_children_with_data('/RC/CLUSTER/%s/GW' % cluster_conf['clusterName'])) 

    # GW lookup
    cluster_conf['gwLookup'] = __znode_to_dict(
            'gwId', util.zk_get_children_with_data('/RC/NOTIFICATION/CLUSTER/%s/GW' % cluster_conf['clusterName'])) 

    return cluster_conf 
예제 #12
0
    def test_zookeeper_delete_root_of_gw_znodes(self):
        util.print_frame()

        # Start load generation
        self.load_gen_list = {}
        arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
        server = self.cluster['servers'][0]
        load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10, verbose=True)
        load_gen.start()
        self.load_gen_list[0] = load_gen

        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Check load
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Delete root of GW znodes
        print 'try remove root of GW znodes'
        ret = util.zk_cmd('rmr /RC/NOTIFICATION/CLUSTER/%s/GW' % server['cluster_name'])
        ret = ret['err']
        self.assertEqual(ret, '', 'failed to remove root of GW znodes, ret:%s' % ret)

        # Check loadbalancing
        for i in range(10):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps > 10:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')

        # Recover root of GW znodes
        print 'try recover GW znodes'
        ret = util.zk_cmd('create /RC/NOTIFICATION/CLUSTER/%s/GW test' % server['cluster_name'])
        ret = ret['err']
        self.assertNotEqual(ret.find('Created /RC/NOTIFICATION/CLUSTER/testCluster0/GW'), -1,
                'failed to create root of GW znodes, ret:%s' % ret)
        for s in self.cluster['servers']:
            path = '/RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (s['cluster_name'], s['id'])
            cmd = 'create %s \'{"ip":"%s","port":%d}\'' % (path, s['ip'], s['gateway_port'])
            print cmd
            ret = util.zk_cmd(cmd)
            ret = ret['err']
            self.assertNotEqual(ret.find('Created %s' % path), -1, 'failed to recover GW znode, ret:%s' % ret)

        # Check loadbalancing
        for i in range(10):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')