def request_to_start_gateway(cluster_name, server, leader_cm, check_state=True): # add gateway configuration to confmaster cmd = 'gw_add %s %d %s %s %d' % (cluster_name, server['id'], server['pm_name'], server['ip'], server['gateway_port']) result = util.cm_command(leader_cm['ip'], leader_cm['cm_port'], cmd) jobj = json.loads(result) if jobj['state'] != 'success': util.log('failed to execute. cmd:%s, result:%s' % (cmd, result)) return -1 # start gateway process ret = util.start_gateway(server['id'], server['ip'], leader_cm['cm_port'], cluster_name, server['gateway_port']) if ret is not 0: util.log('failed to start_gateway. server:%s, id:%d' % (server['ip'], server['id'])) return -1 # check gateway state if check_state: ok = False try_cnt = 0 while try_cnt < 5: try_cnt += 1 if util.check_gateway_state(cluster_name, leader_cm, server): ok = True break time.sleep(0.5) if not ok: util.log('failed to start_gateway, Invalid state of gateway.' % inactive_conns) return -1 # check inactive redis connections of gateway if check_state: ok = False try_cnt = 0 while try_cnt < 10: try_cnt += 1 if util.gw_info_redis_disccons(server['ip'], server['gateway_port']) == 0: ok = True break else: time.sleep(0.5) if not ok: util.log( 'failed to start_gateway, invalid number of inactive redis connections. %d' % inactive_conns) return -1 util.log('succeeded to start_gateway. server:%s, id:%d' % (server['ip'], server['id'])) return 0
def test_gateway_lookup( self ): util.print_frame() # check initial states for server in self.cluster['servers']: success = False for try_cnt in range( 5 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd( cmd ) ret = ret['err'] if -1 != ret.find('cZxid'): success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) # shutdown gateway for server in self.cluster['servers']: success = False ret = util.shutdown_gateway( server['id'], server['gateway_port'] ) self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] ) for try_cnt in range( 10 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd( cmd ) ret = ret['err'] if -1 != ret.find('Node does not exist'): success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) # restart gateway for server in self.cluster['servers']: success = False ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port']) self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] ) for try_cnt in range( 5 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd( cmd ) ret = ret['err'] if -1 != ret.find('cZxid'): success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) return 0
def test_gateway_lookup( self ): util.print_frame() # check initial states for server in self.cluster['servers']: success = False for try_cnt in range( 5 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd(cmd) if ret['exitcode'] == 'OK': success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) # shutdown gateway for server in self.cluster['servers']: success = False ret = util.shutdown_gateway( server['id'], server['gateway_port'] ) self.assertEqual( ret, 0, 'failed : shutdown gateawy%d' % server['id'] ) for try_cnt in range( 10 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd( cmd ) if ret['exitcode'] == 'ZK_NO_NODE': success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) # restart gateway for server in self.cluster['servers']: success = False ret = util.start_gateway( server['id'], server['ip'], self.leader_cm['cm_port'], server['cluster_name'], server['gateway_port']) self.assertEqual( ret, 0, 'failed : start gateawy%d' % server['id'] ) for try_cnt in range( 5 ): cmd = 'get /RC/NOTIFICATION/CLUSTER/%s/GW/%d' % (self.cluster['cluster_name'], server['id']) ret = util.zk_cmd( cmd ) if ret['exitcode'] == 'OK': success = True break time.sleep( 1 ) self.assertEqual( success, True, 'failed : cmd="%s", ret="%s"' % (cmd, ret) ) util.log( 'succeeded : cmd="%s", ret="%s"' % (cmd, ret) ) return 0
def request_to_start_gateway( cluster_name, server, leader_cm, check_state=True ): # add gateway configuration to confmaster cmd = 'gw_add %s %d %s %s %d' % (cluster_name, server['id'], server['pm_name'], server['ip'], server['gateway_port']) result = util.cm_command( leader_cm['ip'], leader_cm['cm_port'], cmd ) jobj = json.loads(result) if jobj['state'] != 'success': util.log('failed to execute. cmd:%s, result:%s' % (cmd, result)) return -1 # start gateway process ret = util.start_gateway( server['id'], server['ip'], leader_cm['cm_port'], cluster_name, server['gateway_port'] ) if ret is not 0: util.log('failed to start_gateway. server:%s, id:%d' % (server['ip'], server['id'])) return -1 # check gateway state if check_state: ok = False try_cnt = 0 while try_cnt < 5: try_cnt += 1 if util.check_gateway_state( cluster_name, leader_cm, server ): ok = True break time.sleep(0.5) if not ok: util.log('failed to start_gateway, Invalid state of gateway.' % inactive_conns) return -1 # check inactive redis connections of gateway if check_state: ok = False try_cnt = 0 while try_cnt < 10: try_cnt += 1 if util.gw_info_redis_disccons(server['ip'], server['gateway_port']) == 0: ok = True break else: time.sleep(0.5) if not ok: util.log('failed to start_gateway, invalid number of inactive redis connections. %d' % inactive_conns) return -1 util.log('succeeded to start_gateway. server:%s, id:%d' % (server['ip'], server['id'])) return 0
def test_gateway_upgrade(self): util.print_frame() api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path) # Start load generation self.load_gen_list = {} for i in range(len(self.cluster['servers'])): arc_api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path) server = self.cluster['servers'][i] load_gen = LoadGenerator_ARCCI(server['id'], arc_api, timeout_second=10) load_gen.start() self.load_gen_list[i] = load_gen # Set up arguments server = self.cluster['servers'][0] gw_id = server['id'] gw_port = server['gateway_port'] # Check load for i in range(5): ok = True for s in self.cluster['servers']: tps = util.get_tps(s['ip'], s['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps)) if tps < 50: ok = False if ok: break time.sleep(1) self.assertTrue(ok, 'failed to send requests') # Delete gateway self.assertTrue( util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT), 'failed to delete gateway') # Check load for i in range(5): ok = True tps = util.get_tps(server['ip'], server['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps)) if tps > 10: ok = False if ok: break time.sleep(1) self.assertTrue(ok, 'failed to send requests') # Stop gateway ret = util.shutdown_gateway(gw_id, gw_port, True) self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id) # Check if gateway is deleted deleted_gw = {"ip":HOST_IP,"port":gw_port} found = check_gateway_deleted(deleted_gw, api) if not found: self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw)) else: util.log('SUCCESS, delete gateway information.') # Start gateway ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port) self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id ) time.sleep(3) # Add gateway self.assertTrue( util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT), 'failed to add gateway') # Check if gateway is added added_gw = {"ip":HOST_IP,"port":gw_port} log_reader = LogReader(api.conf.log_file_prefix) found = False while True: line = log_reader.readline() if line == None: break if line.find(MSG_GATEWAY_ADD_ZK) == -1: continue gw = line.split('data:')[1] gw = ast.literal_eval(gw) if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']: found = True if not found: self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw)) else: util.log('SUCCESS, load gateway information.') # Check loadbalancing for i in range(5): ok = True for s in self.cluster['servers']: tps = util.get_tps(s['ip'], s['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps)) if tps < 50: ok = False if ok: break time.sleep(1) if not ok: self.fail('FAIL, loadbalancing,') else: util.log('SUCCESS, loadbalancing.') api.destroy()
def _gateway_fault_failback(self, api): # Set up arguments server = self.cluster['servers'][0] gw_id = server['id'] gw_port = server['gateway_port'] # Stop gateway ret = util.shutdown_gateway(gw_id, gw_port, True) self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id) time.sleep(3) # Check error saved_err_cnt = {} for i in range(len(self.load_gen_list)): saved_err_cnt[i] = 0 no_error_cnt = 0 for i in range(10): util.log('check error count loop:%d' % i) no_err = True for j in range(len(self.load_gen_list)): err_cnt = self.load_gen_list[j].get_err_cnt() if err_cnt != saved_err_cnt[j]: no_err = False util.log('saved_err_cnt:%d, err_cnt:%d' % (saved_err_cnt[j], err_cnt)) saved_err_cnt[j] = err_cnt if no_err: no_error_cnt += 1 if no_error_cnt >= 3: break time.sleep(1) self.assertTrue(no_error_cnt >= 3, 'failed to get replys from well working gateways') # Check if gateway is deleted deleted_gw = {"ip":HOST_IP,"port":gw_port} found = check_gateway_deleted(deleted_gw, api) if not found: self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw)) else: util.log('SUCCESS, delete gateway information.') # Start gateway ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port) self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id ) time.sleep(3) # Check if gateway is added found = check_gateway_added(deleted_gw, api) if not found: self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw)) else: util.log('SUCCESS, load gateway information.') # Check loadbalancing for i in range(3): ok = True for s in self.cluster['servers']: tps = util.get_tps(s['ip'], s['gateway_port'], 'gw') if tps < 50: ok = False if ok: break time.sleep(1) if not ok: self.fail('FAIL, loadbalancing,') else: util.log('SUCCESS, loadbalancing.')
def test_gateway_add_del(self): util.print_frame() api = ARC_API(ZK_ADDR, CLUSTER_NAME, logFilePrefix = self.arcci_log, so_path = self.so_path) # Add gateway gw_port = 10000 gw_id = 10 cmd = 'gw_add %s %d %s %s %d' % (CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port) ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd) if ret != None and len(ret) >= 2: ret = ret[:-2] util.log('cmd:"%s", ret:"%s"' % (cmd, ret)) if not ret.startswith('{"state":"success","msg":"+OK"}'): self.fail('failed to add gateway') # Deploy gateway server = self.cluster['servers'][0] ret = util.deploy_gateway(gw_id) self.assertTrue(ret, 'failed to deploy_gateway') # Start gateway ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port) self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id ) time.sleep(5) # Check if gateway is added added_gw = {"ip":HOST_IP,"port":gw_port} log_reader = LogReader(api.conf.log_file_prefix) found = False while True: line = log_reader.readline() if line == None: break if line.find(MSG_GATEWAY_ADD_ZK) == -1: continue gw = line.split('data:')[1] gw = ast.literal_eval(gw) if gw['ip'] == added_gw['ip'] and gw['port'] == added_gw['port']: found = True if not found: self.fail('FAIL, load gateway information, gw:%s' % util.json_to_str(added_gw)) else: util.log('SUCCESS, load gateway information.') # Delete gateway cmd = 'gw_del %s %d' % (CLUSTER_NAME, gw_id) ret = util.cm_command(MGMT_IP, MGMT_PORT, cmd) if ret != None and len(ret) >= 2: ret = ret[:-2] util.log('cmd:"%s", ret:"%s"' % (cmd, ret)) if not ret.startswith('{"state":"success","msg":"+OK"}'): self.fail('failed to delete gateway') # Check if gateway is deleted deleted_gw = {"ip":HOST_IP,"port":gw_port} found = check_gateway_deleted(deleted_gw, api) if not found: self.fail('FAIL, delete gateway information, gw:%s' % util.json_to_str(deleted_gw)) else: util.log('SUCCESS, delete gateway information.') # Stop gateway ret = util.shutdown_gateway(gw_id, gw_port) self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id) api.destroy()
def test_no_error_scenario_and_memory_leak(self): util.print_frame() # Start with valgrind p = util.exec_proc_async('%s/.obj%d' % (constant.ARCCI_DIR, self.arch), "valgrind ./dummy-perf -z localhost:2181 -c %s -n 5 -s 60" % self.cluster['cluster_name'], subprocess.PIPE, subprocess.PIPE, subprocess.PIPE); # Set up arguments server = self.cluster['servers'][0] gw_id = server['id'] gw_port = server['gateway_port'] # Check load for i in range(5): ok = True for s in self.cluster['servers']: tps = util.get_tps(s['ip'], s['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps)) if tps < 50: ok = False if ok: break time.sleep(1) self.assertTrue(ok, 'failed to send requests') # Delete gateway self.assertTrue( util.gw_del(CLUSTER_NAME, gw_id, MGMT_IP, MGMT_PORT), 'failed to delete gateway') # Check load for i in range(5): ok = True tps = util.get_tps(server['ip'], server['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (server['ip'], server['gateway_port'], tps)) if tps > 10: ok = False if ok: break time.sleep(1) self.assertTrue(ok, 'failed to send requests') # Stop gateway ret = util.shutdown_gateway(gw_id, gw_port, True) self.assertEqual(ret, 0, 'failed : shutdown gateawy%d' % gw_id) time.sleep(5) # Start gateway ret = util.start_gateway( gw_id, server['ip'], MGMT_PORT, server['cluster_name'], gw_port) self.assertEqual( ret, 0, 'failed : start gateawy%d' % gw_id ) time.sleep(3) # Add gateway self.assertTrue( util.gw_add(CLUSTER_NAME, gw_id, HOST_NAME, HOST_IP, gw_port, MGMT_IP, MGMT_PORT), 'failed to add gateway') time.sleep(10) # Check loadbalancing for i in range(5): ok = True for s in self.cluster['servers']: tps = util.get_tps(s['ip'], s['gateway_port'], 'gw') util.log('%s:%d TPS:%d' % (s['ip'], s['gateway_port'], tps)) if tps < 50: ok = False if ok: break time.sleep(1) if not ok: self.fail('FAIL, loadbalancing,') else: util.log('SUCCESS, loadbalancing.') # Check no error no_memory_leak = False (stdout, stderr) = p.communicate() for line in stdout.split("\n"): print line if line.find('[ERR]') != -1: self.fail('find error, msg:%s' % line) if line.find('All heap blocks were freed -- no leaks are possible'): no_memory_leak = True self.assertTrue(no_memory_leak, 'memory leaks are possible') util.log('no leaks are possible')