예제 #1
0
    def test_gateway_network_isolation(self):
        util.print_frame()

        cluster = self.cluster

        # Clear rules
        while True:
            out = util.sudo('iptables -t nat -D OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
            util.log(out)
            if out.succeeded == False:
                break

        while True:
            out = util.sudo('iptables -t nat -D PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
            util.log(out)
            if out.succeeded == False:
                break

        while True:
            out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
            util.log(out)
            if out.succeeded == False:
                break

        # Print rules
        out = util.sudo('iptables -L')
        util.log('====================================================================')
        util.log(out.succeeded)
        util.log('out : %s' % out)
        util.log('out.return_code : %d' % out.return_code)
        util.log('out.stderr : %s' % out.stderr)

        # Start loadgenerators
        self.load_gen_list = {}
        for i in range(len(cluster['servers'])):
            arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
            server = cluster['servers'][i]
            load_gen = LoadGenerator_ARCCI_FaultTolerance(server['id'], arc_api)
            load_gen.start()
            self.load_gen_list[i] = load_gen

        # Add forwarding role (127.0.0.100 -> 127.0.0.1)
        out = util.sudo('iptables -t nat -A OUTPUT -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
        self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)

        out = util.sudo('iptables -t nat -A PREROUTING -d 127.0.0.100 -p tcp -j DNAT --to-destination 127.0.0.1')
        self.assertTrue(out.succeeded, 'add a forwarding role to iptables fail. output:%s' % out)

        # Add virtualhost information to MGMT
        VIRTUAL_HOST_NAME = 'virtualhost'
        VIRTUAL_HOST_IP = '127.0.0.100'
        ret = util.pm_add(VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'pm_add fail.')

        # Modify gateway information of MGMT
        server = cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Delete gateway
        ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_del fail')

        # Add gateway
        ret= util.gw_add(CLUSTER_NAME, gw_id, VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, gw_port, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_add fail')

        # Check load balancing
        for i in range(5):
            ok = True
            for s in cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail')
        util.log('load balancing success')

        # Block
        out = util.sudo('iptables -A OUTPUT -d 127.0.0.100 -j DROP')
        self.assertTrue(out.succeeded, 'add a bloking role to iptables fail. output:%s' % out)

        # Check blocked gateway`s ops
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - blocked gateway')
        util.log('load balancing success - blocked gateway')

        # Check unblocked gateway`s ops
        for i in range(10):
            ok = True
            for s in cluster['servers']:
                if s == server:
                    continue

                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - nonblocked gateways')
        util.log('load balancing success - nonblocked gateways')

        # Unblock
        out = util.sudo('iptables -D OUTPUT -d 127.0.0.100 -j DROP')
        self.assertTrue(out.succeeded, 'delete a bloking role to iptables fail. output:%s' % out)

        # Check load balancing
        ok = False
        for i in xrange(5):
            condition = (lambda s: (s['ops'] <= 10 if s['id'] == gw_id else s['ops'] >= 50))
            if util.check_ops(cluster['servers'], 'gw', condition):
                ok = True
                break
            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
        util.log('load balancing success - all gateways after unblocking network')

        server = cluster['servers'][0]

        # Wait until opinion for the gateway deleted.
        for i in xrange(5):
            util.log('Wait until opinions for the gateway have been deleted... %d' % i)
            time.sleep(1)

        # Delete gateway
        ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_del fail')

        # Add gateway
        ret= util.gw_add(CLUSTER_NAME, gw_id, server['pm_name'], server['ip'], gw_port, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_add fail')

        # Check load balancing
        for i in range(10):
            ok = True
            for s in cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
        util.log('load balancing success - all gateways after unblocking network')
예제 #2
0
    def test_no_error_scenario_and_memory_leak(self):
        util.print_frame()

        # Start with valgrind
        p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch),
                "valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'],
                subprocess.PIPE, subprocess.PIPE, subprocess.PIPE);

        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Check load
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Delete gateway
        self.assertTrue(
                util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
                'failed to delete gateway')

        # Check load
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port, True)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)
        time.sleep(5)

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
        time.sleep(3)

        # Add gateway
        self.assertTrue(
                util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
                'failed to add gateway')
        time.sleep(10)

        # Check loadbalancing
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')

        # Check no error
        no_memory_leak = False
        (stdout, stderr) = p.communicate()
        for line in stdout.split("\n"):
            print line

            if line.find('[ERR]') != -1:
                self.fail('find error, msg:%s' % line)

            if line.find('All heap blocks were freed -- no leaks are possible'):
                no_memory_leak = True

        self.assertTrue(no_memory_leak, 'memory leaks are possible')
        util.log('no leaks are possible')
예제 #3
0
    def test_gateway_upgrade(self):
        util.print_frame()

        api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)

        # Start load generation
        self.load_gen_list = {}
        for i in range(len(self.cluster['servers'])):
            arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
            server = self.cluster['servers'][i]
            load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10)
            load_gen.start()
            self.load_gen_list[i] = load_gen

        # Set up arguments
        server = self.cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Check load
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Delete gateway
        self.assertTrue(
                util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT),
                'failed to delete gateway')

        # Check load
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)
        self.assertTrue(ok, 'failed to send requests')

        # Stop gateway
        ret = util.shutdown_gateway(gw_id, gw_port, True)
        self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id)

        # Check if gateway is deleted
        deleted_gw = {"ip":HOST_IP,"port":gw_port}

        found = check_gateway_deleted(deleted_gw, api)
        if not found:
            self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw))
        else:
            util.log('SUCCESS, delete gateway information.')

        # Start gateway
        ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port)
        self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id )
        time.sleep(3)

        # Add gateway
        self.assertTrue(
                util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT),
                'failed to add gateway')

        # Check if gateway is added
        added_gw = {"ip":HOST_IP,"port":gw_port}

        log_reader = LogReader(api.conf.log_file_prefix)
        found = False
        while True:
            line = log_reader.readline()
            if line == None:
                break

            if line.find(MSG_GATEWAY_ADD_ZK) == -1:
                continue

            gw = line.split('data:')[1]
            gw = ast.literal_eval(gw)

            if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']:
                found = True

        if not found:
            self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw))
        else:
            util.log('SUCCESS, load gateway information.')

        # Check loadbalancing
        for i in range(5):
            ok = True
            for s in self.cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        if not ok:
            self.fail('FAIL, loadbalancing,')
        else:
            util.log('SUCCESS, loadbalancing.')

        api.destroy()
예제 #4
0
    def test_gateway_network_isolation(self):
        util.print_frame()

        cluster = self.cluster

        # Clear rules
        while True:
            if util.iptables_redirect('D', '127.0.0.100', '127.0.0.1') == False:
                break

        while True:
            if util.iptables_drop('D', '127.0.0.100') == False:
                break

        # Print rules
        util.iptables_print_list()

        # Start loadgenerators
        self.load_gen_list = {}
        for i in range(len(cluster['servers'])):
            arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path)
            server = cluster['servers'][i]
            load_gen = LoadGenerator_ARCCI_FaultTolerance(server['id'], arc_api)
            load_gen.start()
            self.load_gen_list[i] = load_gen

        # Add forwarding role (127.0.0.100 -> 127.0.0.1)
        self.assertTrue(util.iptables_redirect('A', '127.0.0.100', '127.0.0.1'), 'add a forwarding role to iptables fail.')

        # Add virtualhost information to MGMT
        VIRTUAL_HOST_NAME = 'virtualhost'
        VIRTUAL_HOST_IP = '127.0.0.100'
        ret = util.pm_add(VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'pm_add fail.')

        # Modify gateway information of MGMT
        server = cluster['servers'][0]
        gw_id = server['id']
        gw_port = server['gateway_port']

        # Delete gateway
        ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_del fail')

        # Add gateway
        ret= util.gw_add(CLUSTER_NAME, gw_id, VIRTUAL_HOST_NAME, VIRTUAL_HOST_IP, gw_port, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_add fail')

        # Check load balancing
        for i in range(5):
            ok = True
            for s in cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail')
        util.log('load balancing success')

        # Block
        self.assertTrue(util.iptables_drop('A', '127.0.0.100'), 'add a bloking role to iptables fail.')

        # Check blocked gateway`s ops
        for i in range(5):
            ok = True
            tps = util.get_tps(server['ip'], server['gateway_port'], 'gw')
            util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps))
            if tps > 10:
                ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - blocked gateway')
        util.log('load balancing success - blocked gateway')

        # Check unblocked gateway`s ops
        for i in range(10):
            ok = True
            for s in cluster['servers']:
                if s == server:
                    continue

                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - nonblocked gateways')
        util.log('load balancing success - nonblocked gateways')

        # Unblock
        self.assertTrue(util.iptables_drop('D', '127.0.0.100'), 'delete a bloking role to iptables fail.')

        # Check load balancing
        ok = False
        for i in xrange(5):
            condition = (lambda s: (s['ops'] <= 10 if s['id'] == gw_id else s['ops'] >= 50))
            if util.check_ops(cluster['servers'], 'gw', condition):
                ok = True
                break
            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
        util.log('load balancing success - all gateways after unblocking network')

        server = cluster['servers'][0]

        # Wait until opinion for the gateway deleted.
        for i in xrange(5):
            util.log('Wait until opinions for the gateway have been deleted... %d' % i)
            time.sleep(1)

        # Delete gateway
        ret = util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_del fail')

        # Add gateway
        ret= util.gw_add(CLUSTER_NAME, gw_id, server['pm_name'], server['ip'], gw_port, MGMT_IP, MGMT_PORT)
        self.assertTrue(ret, 'gw_add fail')

        # Check load balancing
        for i in range(10):
            ok = True
            for s in cluster['servers']:
                tps = util.get_tps(s['ip'], s['gateway_port'], 'gw')
                util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps))
                if tps < 50:
                    ok = False

            if ok:
                break

            time.sleep(1)

        self.assertTrue(ok, 'load balancing fail - all gateways after unblocking network')
        util.log('load balancing success - all gateways after unblocking network')

        # Go back to initial configuration
        self.assertTrue(util.pm_del(MGMT_IP, MGMT_PORT, VIRTUAL_HOST_NAME), 
                'failed to pm_del to go back to initial configuration.')