Esempio n. 1
0
def request_to_shutdown_gateway( cluster_name, server, leader_cm, check=False ):
    ip = server['ip']
    port = server['gateway_port']
    id = server['id']

    # delete gateway configuration from confmaster
    cmd = 'gw_del %s %d' % (cluster_name, id)
    result = util.cm_command( leader_cm['ip'], leader_cm['cm_port'], cmd )
    jobj = json.loads(result)
    if jobj['state'] != 'success':
        util.log('failed to execute. cmd:%s, result:%s' % (cmd, result))
        return -1

    # check client connection
    ok = False
    for i in range(10):
        client_conn_num = util.gw_info_client_cnt(ip, port)
        if client_conn_num == 1:
            ok = True
            break
        else:
            time.sleep(1)

    if ok == False:
        util.log('failed to shutdown_gateway, invalid number of client connections. %d' % (client_conn_num - 1))
        return -1

    # shutdown gateway process
    if util.shutdown_gateway( id, port ) is not 0:
        util.log('failed to shutdown_gateway %d' % (id))
        return -1
    util.log('succeeded to shutdown_gateway. %d' % (id))
    return 0
Esempio n. 2
0
    def test_gateway_lookup( self ):
        util.print_frame()

        # check initial states
        for server in self.cluster['servers']:
            success = False
            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('cZxid'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # shutdown gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.shutdown_gateway( server['id'], server['gateway_port'] )
            self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] )

            for try_cnt in range( 10 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('Node does not exist'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        # restart gateway
        for server in self.cluster['servers']:
            success = False

            ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port'])
            self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] )

            for try_cnt in range( 5 ):
                cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id'])
                ret = util.zk_cmd( cmd )
                ret = ret['err']
                if -1 != ret.find('cZxid'):
                    success = True
                    break
                time.sleep( 1 )

            self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) )
            util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) )

        return 0
Esempio n. 3
0
    def test_gateway_upgrade(self):
        util.print_frame()

        api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)

        # Start load generation
        self.load_gen_list = {}
        for i in range(len(self.cluster['servers'])):
            arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
            server = self.cluster['servers'][i]
            load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10)
            load_gen.start()
            self.load_gen_list[i] = load_gen

        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Check load
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Delete gateway
        self.assertTrue(
                util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
                'failed to delete gateway')

        # Check load
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port, True)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)

        # Check if gateway is deleted
        deleted_gw = {"ip":HOST_IP,"port":gw_port}

        found = check_gateway_deleted(deleted_gw, api)
        if not found:
            self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
        else:
            util.log('SUCCESS, delete gateway information.')

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
        time.sleep(3)

        # Add gateway
        self.assertTrue(
                util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
                'failed to add gateway')

        # Check if gateway is added
        added_gw = {"ip":HOST_IP,"port":gw_port}

        log_reader = LogReader(api.conf.log_file_prefix)
        found = False
        while True:
            line = log_reader.readline()
            if line == None:
                break

            if line.find(MSG_GATEWAY_ADD_ZK) == -1:
                continue

            gw = line.split('data:')[1]
            gw = ast.literal_eval(gw)

            if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
                found = True

        if not found:
            self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
        else:
            util.log('SUCCESS, load gateway information.')

        # Check loadbalancing
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')

        api.destroy()
Esempio n. 4
0
    def _gateway_fault_failback(self, api):
        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port, True)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
        time.sleep(3)

        # Check error
        saved_err_cnt = {}
        for i in range(len(self.load_gen_list)):
            saved_err_cnt[i] = 0

        no_error_cnt = 0
        for i in range(10):
            util.log('check error count loop:%d' % i)
            no_err = True
            for j in range(len(self.load_gen_list)):
                err_cnt = self.load_gen_list[j].get_err_cnt()
                if err_cnt != saved_err_cnt[j]:
                    no_err = False
                util.log('saved_err_cnt:%d, err_cnt:%d' % (saved_err_cnt[j], err_cnt))
                saved_err_cnt[j] = err_cnt

            if no_err:
                no_error_cnt += 1

            if no_error_cnt >= 3:
                break

            time.sleep(1)
        self.assertTrue(no_error_cnt >= 3, 'failed to get replys from well working gateways')

        # Check if gateway is deleted
        deleted_gw = {"ip":HOST_IP,"port":gw_port}

        found = check_gateway_deleted(deleted_gw, api)
        if not found:
            self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
        else:
            util.log('SUCCESS, delete gateway information.')

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
        time.sleep(3)

        # Check if gateway is added
        found = check_gateway_added(deleted_gw, api)
        if not found:
            self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
        else:
            util.log('SUCCESS, load gateway information.')

        # Check loadbalancing
        for i in range(3):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')
Esempio n. 5
0
    def test_gateway_add_del(self):
        util.print_frame()

        api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)

        # Add gateway
        gw_port = 10000
        gw_id = 10
        cmd = 'gw_add %s %d %s %s %d' % (CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port)
        ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
        if ret != None and len(ret) >= 2:
            ret = ret[:-2]
        util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
        if not ret.startswith('{"state":"success","msg":"+OK"}'):
            self.fail('failed to add gateway')

        # Deploy gateway
        server = self.cluster['servers'][0]
        ret = util.deploy_gateway(gw_id)
        self.assertTrue(ret, 'failed to deploy_gateway')

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )

        time.sleep(5)

        # Check if gateway is added
        added_gw = {"ip":HOST_IP,"port":gw_port}

        log_reader = LogReader(api.conf.log_file_prefix)
        found = False
        while True:
            line = log_reader.readline()
            if line == None:
                break

            if line.find(MSG_GATEWAY_ADD_ZK) == -1:
                continue

            gw = line.split('data:')[1]
            gw = ast.literal_eval(gw)

            if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
                found = True

        if not found:
            self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
        else:
            util.log('SUCCESS, load gateway information.')

        # Delete gateway
        cmd = 'gw_del %s %d' % (CLUSTER_NAME, gw_id)
        ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd)
        if ret != None and len(ret) >= 2:
            ret = ret[:-2]
        util.log('cmd:"%s", ret:"%s"' % (cmd, ret))
        if not ret.startswith('{"state":"success","msg":"+OK"}'):
            self.fail('failed to delete gateway')

        # Check if gateway is deleted
        deleted_gw = {"ip":HOST_IP,"port":gw_port}

        found = check_gateway_deleted(deleted_gw, api)
        if not found:
            self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
        else:
            util.log('SUCCESS, delete gateway information.')

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)

        api.destroy()
Esempio n. 6
0
    def test_no_error_scenario_and_memory_leak(self):
        util.print_frame()

        # Start with valgrind
        p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch),
                "valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'],
                subprocess.PIPE, subprocess.PIPE, subprocess.PIPE);

        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Check load
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Delete gateway
        self.assertTrue(
                util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
                'failed to delete gateway')

        # Check load
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port, True)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
        time.sleep(5)

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
        time.sleep(3)

        # Add gateway
        self.assertTrue(
                util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
                'failed to add gateway')
        time.sleep(10)

        # Check loadbalancing
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')

        # Check no error
        no_memory_leak = False
        (stdout, stderr) = p.communicate()
        for line in stdout.split("\n"):
            print line

            if line.find('[ERR]') != -1:
                self.fail('find error, msg:%s' % line)

            if line.find('All heap blocks were freed -- no leaks are possible'):
                no_memory_leak = True

        self.assertTrue(no_memory_leak, 'memory leaks are possible')
        util.log('no leaks are possible')
    def test_7_remaining_hbc_connection( self ):
        util.print_frame()

        # check pgs
        for server in self.cluster['servers']:
            before_cnt_redis = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
            before_cnt_smr = util.get_clients_count_of_smr(server['smr_mgmt_port'])

            cmd = 'pgs_leave %s %d forced\r\npgs_del %s %d' % (self.cluster['cluster_name'], server['id'], self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            # check redis
            success = False
            for i in range(5):
                after_cnt = util.get_clients_count_of_redis(server['ip'], server['redis_port'])
                if after_cnt <= 2:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )
            util.log( 'succeeded : the number of connections to redis%d(%s:%d) is %d, exptected=n<=2, before=%d' % (server['id'], server['ip'], server['redis_port'], after_cnt, before_cnt_redis) )

            # check smr
            success = False
            expected = 0
            for i in range(5):
                after_cnt = util.get_clients_count_of_smr(server['smr_mgmt_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)
            self.assertEquals( success, True, 'failed : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )
            util.log( 'succeeded : the number of connections to smr%d(%s:%d) is %d, exptected=%d, before=%d' % (server['id'], server['ip'], server['smr_mgmt_port'], after_cnt, expected, before_cnt_smr) )

        # check gateway
        for server in self.cluster['servers']:
            before_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])

            cmd = 'gw_del %s %d' % (self.cluster['cluster_name'], server['id'])
            util.cm_command(self.leader_cm['ip'], self.leader_cm['cm_port'], cmd)

        for server in self.cluster['servers']:
            success = False
            expected = 1
            for i in range(5):
                after_cnt = util.get_clients_count_of_gw(server['ip'], server['gateway_port'])
                if after_cnt == expected:
                    success = True
                    break
                time.sleep(1)

            self.assertEquals( success, True, 'failed : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )
            util.log( 'succeeded : the number of connections to gateway%d(%s:%d) is %d, exptected=%d.' % (server['id'], server['ip'], server['gateway_port'], after_cnt, expected) )

        # Go back to initial configuration
        # Cleanup PG
        self.assertTrue(util.cm_success(util.cm_command(
            self.leader_cm['ip'], self.leader_cm['cm_port'], 
            'pg_del %s %d' % (self.cluster['cluster_name'], self.cluster['servers'][0]['pg_id'])))[0])

        # Cleanup processes of PGS and GW
        for s in self.cluster['servers']:
            self.assertEqual(0, util.shutdown_redis(s['id'], s['redis_port']), 
                'failed to kill redis %d process' % s['id'])
            self.assertEqual(0, util.shutdown_smr(s['id'], s['ip'], s['smr_base_port']), 
                'failed to kill smr %d process' % s['id'])
            self.assertEqual(0, util.shutdown_gateway(s['id'], s['gateway_port']),
                'failed to kill gw %d process' % s['id'])

        # Recover PG
        self.assertTrue(
                util.install_pg(self.cluster, self.cluster['servers'], self.cluster['servers'][0], start_gw=True),
                'failed to recover PGS and GW in a PM')