def is_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0): for now in utils.run_till_timeout(time_out, ping_timeout): reachable = dest_is_reachable(ssh_client, dest_ip) if reachable: return True mesg = ("Dest[%s] not-reachable retry in %s seconds." % (dest_ip, time_out)) utils.log_msg(mesg) return False
def run_l7_switching(on_server_name_list, lb_vip_address, url_path='', count=4): resp_urls = lbaas2.count_http_servers(lb_vip_address, count=count, url_path=url_path) if not url_responses_are_OK(resp_urls, on_server_name_list): utils.log_msg("http://%s/%s redirected to wrong pool" % ( lb_vip_address, url_path), "LBaaS-L7 ERROR") return False return True
def isnot_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0, idle_time=2.0): if idle_time > 0.0: time.sleep(idle_time) for now in utils.run_till_timeout(time_out, ping_timeout): reachable = dest_is_reachable(ssh_client, dest_ip) if not reachable: return True mesg = ("Dest[%s] is reachable retry in %s seconds." % (dest_ip, time_out)) utils.log_msg(mesg) return False
def dest_is_reachable(ssh_client, dest_ip): XPTN = r"(\d+).*transmit.*(\d+).*receive.*(\d+).*loss" try: result = ssh_client.ping_host(dest_ip) utils.log_msg(result) m = re.search(XPTN, result, (re.I | re.M)) if m and int(m.group(1)) > 0 and int(m.group(3)) == 0: return True else: return False except Exception: tb_str = traceback.format_exc() mesg = ("ERROR test dest_ip[%s] is reachable:\n%s" % ( dest_ip, tb_str)) utils.log_msg(mesg) return False
def delete_the_loadbalancer(cmgr, lb_id, waitfor_delete=800, pause_before_wait=2.0, interval=5.0): cmgr.lbaas("loadbalancer-delete", lb_id) time_start = time.time() end_time = time.time() + waitfor_delete time.sleep(pause_before_wait) while (time.time() < end_time): lb_list = cmgr.lbaas("loadbalancer-list", id=lb_id) if len(lb_list) == 0: break time.sleep(interval) etime = time.time() - time_start utils.log_msg("elapse-time,delete-loadbalancer[%s]" % etime, "OS-Stats") return cmgr.lbaas("loadbalancer-list", id=lb_id)
def create_solaris(adm, os_auth_url, halt=False, password="******"): if halt: import pdb; pdb.set_trace() # get-or-create Sun solaris system's admin Sun try: tenant = utils.fgrep(adm.keys('tenant-list'), name=r'^Sun$') if len(tenant) < 1: Sun = icreds.create_admin_project('Sun', password) sun = utils.get_mimic_manager_cli(os_auth_url, 'Sun', password) except Exception: tb_str = traceback.format_exc() mesg = ("ERROR creating/retriving Admin user[%s]:\n%s" % ( 'Sun', tb_str)) utils.log_msg(mesg) # our solar system has 8 planets' sun_planets = ['Mercury', 'Venus', 'Earth', 'Mars', 'Jupiter', 'Satun', 'Uranus', 'Neptune'] dwarf_planets = ["Haumea", "Eris", "Ceres", "Pluto", "Makemake"] tenants = {} for planet in sun_planets + dwarf_planets + ["Moon"]: tenant = utils.fgrep(adm.keys('tenant-list'), name=planet) if len(tenant) < 1: # tenant not exist, create it; default password=itempest tenant = icreds.create_primary_project(planet, password) try: tenant = adm.keys('tenant_get_by_name', planet) tenants[planet] = tenant # by default tenant can only have instances=10 adm.nova('quota-update', tenant['id'], instances=tenant_max_instances) adm.qsvc('quota-incr-by', tenant['id'], 2) except Exception: pass else: tenants[planet] = tenant[0] return tenants
def add_floatingip_to_server_and_test_reachibility( cli_mgr, server_id, security_group_id=None, action='add', check_interval=5, check_duration=60): mesg_p = "%s reach server[%s %s] with-ip[%s %s]." try: ss = cli_mgr.nova('server-show', server_id) fip_result = add_floatingip_to_server( cli_mgr, server_id, security_group_id=security_group_id, check_interval=check_interval, check_duration=check_duration, action=action) m_type = "CAN" if fip_result[1] else "CAN'T" mesg = mesg_p % (m_type, server_id, ss['name'], fip_result[0]['fixed_ip_address'], fip_result[0]['floating_ip_address']) utils.log_msg(mesg) return fip_result except Exception: tb_str = traceback.format_exc() mesg = ("ERROR creating floatingip for server[%s]:\n%s" % ( server_id, tb_str)) utils.log_msg(mesg) return (server_id, False)
def poke_http_stats_change(nsxt_client, cmgr, lb2_name, interval=5.0, poke_count=100, no_http_traffic=False): lb2_fip = lbaas2.get_loadbalancer_floatingip(cmgr, lb2_name) lb2_web_ip = lb2_fip[1]['floating_ip_address'] lb = cmgr.lbaas('loadbalancer-show', lb2_name) lb_port = cmgr.qsvc('port-show', lb['vip_port_id']) os_security_group_id = lb_port['security_groups'][0] _sgroup = cmgr.qsvc('security-group-show', os_security_group_id) msg = "os-security-group-id = %s" % (os_security_group_id) utils.log_msg(msg, 'CHK-Stats') sg_rules = _sgroup['security_group_rules'] http_rule = [x for x in sg_rules if x['port_range_min'] >= 80][0] filters = {'os-neutron-secgr-id': os_security_group_id} fw_list = nsxt_client.list_firewall_sections(**filters) sect_id = fw_list[0].get('id') fw_rules = nsxt_client.get_firewall_section_rules(sect_id) rule = [x for x in fw_rules if x['display_name'] == http_rule.get('id')] rule_id = rule[0].get('id') msg = "NSX STATS is at section[%s] rule_id[%s]" % (sect_id, rule_id) utils.log_msg(msg, 'CHK-Stats') nsx_stats = nsxt_client.get_firewall_section_rule_stats(sect_id, rule_id) os_stats1 = log_lb2_stats(cmgr, lb2_name) if no_http_traffic: utils.log_msg("No http traffic initiated", 'CHK-Stats') else: lbaas2.count_http_servers(lb2_web_ip) m_stats_change(nsxt_client, sect_id, rule_id, nsx_stats, cmgr, lb2_name, interval=interval, poke_count=poke_count) os_stats2 = log_lb2_stats(cmgr, lb2_name) dd = {} for k in os_stats2.keys(): dd[k] = int(os_stats2[k]) - int(os_stats1[k]) utils.log_msg("%s" % str(dd), "OS-STATS-DIFF")
def m_stats_change(nsxt_client, sect_id, rule_id, nsx_stats, cmgr, lb2_name, interval=2.5, poke_count=150): t0 = time.time() for cnt in range(poke_count): log_lb2_stats(cmgr, lb2_name) ss = nsxt_client.get_firewall_section_rule_stats(sect_id, rule_id) ss.pop('_schema', None) ss.pop('rule_id', None) msg = "#%02d %s" % (cnt, str(ss)) utils.log_msg(msg, 'CHK-Stats') if ss.get('session_count') != nsx_stats.get('session_count'): e_time = (time.time() - t0) msg = "Take %d seconds for STATS to be updated" % int(e_time) utils.log_msg(msg, 'CHK-Stats') dd = {} for k in ss.keys(): dd[k] = int(ss[k]) - int(nsx_stats[k]) utils.log_msg("%s" % str(dd), "NSX-STATS-DIFF") return e_time time.sleep(interval) tt = (time.time() - t0) raise Exception("After %d seconds, stats do not change!" % int(tt))
def show_toplogy(cli_mgr, return_topo=False, prefix=None, router_id=None, delete_resources=False): tenant_name = cli_mgr.manager.credentials.tenant_name FMT_ROUTER = "%s>> {router_type} router: {name} {id}" % (' ' * 2) FMT_ROUTER_O = "%s>> router: {name} {id}" % (' ' * 2) FMT_X_GW1 = "%sGW: snat_enabled: {enable_snat}" % (' ' * 5) FMT_X_GW2 = "%sfixed_ip: {external_fixed_ips}" % (' ' * (5 + 4)) FMT_X_ROUT = "%sroutes: {routes}" % (' ' * (5 + 4)) FMT_INTERFACE = "%s>> interface: {name} {id}" % (' ' * 6) # FMT_SUBNETS = "%s subnets: {subnets}" % (' ' * 8) FMT_SNET_ADDR = "%s subnet: {id} {name} cidr={cidr} gw={gateway_ip}" % ( ' ' * 8) FMT_SERVER = "%s>> server: {name} {id}" % (' ' * 10) FMT_SERV_ADDR = "%s>> network: %s " topo = [] topo_line = ["\nNetwork topology of tenant[%s]" % tenant_name] s_list = cli_mgr.nova('server-list-with-detail') if router_id: router_list = cli_mgr.qsvc('router-list', id=router_id) else: router_list = cli_mgr.qsvc('router-list') if prefix: ser_name = "^%s" % prefix s_list = utils.fgrep(s_list, name=ser_name) router_list = utils.fgrep(router_list, name=ser_name) sorted(router_list, key=itemgetter('name')) for router in router_list: rtr = _g_by_attr(router, ('id', 'name', 'router_type', 'distributed')) rtr['_networks'] = [] if 'distributed' in router and router['distributed']: rtr['router_type'] = 'distributed' try: topo_line.append(FMT_ROUTER.format(**rtr)) except: topo_line.append(FMT_ROUTER_O.format(**rtr)) if type(router['external_gateway_info']) is dict: xnet_info = router['external_gateway_info'] rtr['gateway'] = xnet_info topo_line.append(FMT_X_GW1.format(**xnet_info)) try: topo_line.append(FMT_X_GW2.format(**xnet_info)) except: utils.log_msg("GW does not have external_fixed_ips: %s" % xnet_info) rtr['routes'] = router['routes'] if (type(router['routes']) in [list, tuple] and len(router['routes']) > 0): topo_line.append(FMT_X_ROUT.format(**router)) rp_list = cli_mgr.qsvc('router-port-list', router['id']) for rp in rp_list: network = cli_mgr.qsvc('net-show', rp['network_id']) netwk = _g_by_attr(network, ('name', 'id')) subnet_list = network['subnets'] netwk['port_id'] = rp['id'] netwk['_servers'] = [] topo_line.append(FMT_INTERFACE.format(**netwk)) netwk['subnets'] = [] for subnet_id in subnet_list: subnet = cli_mgr.qsvc('subnet-show', subnet_id) subnet = _g_by_attr(subnet, ('id', 'cidr', 'gateway_ip', 'allocation_pools', 'name')) topo_line.append(FMT_SNET_ADDR.format(**subnet)) netwk['subnets'].append(subnet) if_name = network['name'] if_servers = [s for s in s_list if if_name in s['addresses']] for s in if_servers: addr_dict = mdata.get_server_address(s) no_if = len(addr_dict) serv = _g_by_attr(s, ('name', 'id')) serv['#interface'] = no_if topo_line.append( FMT_SERVER.format(**s) + " #interface=%s" % no_if) if if_name in addr_dict: serv['interface'] = addr_dict[if_name] topo_line.append( FMT_SERV_ADDR % (' ' * 14, addr_dict[if_name])) netwk['_servers'].append(serv) rtr['_networks'].append(netwk) topo.append(rtr) print("\n".join(topo_line)) if delete_resources: _delete_topology(cli_mgr, topo) return topo if return_topo else {}
def log_lb2_stats(cmgr, lb2_name): os_stats1 = cmgr.lbaas('loadbalancer-stats', lb2_name) msg = "OS-LB2-Stats %s" % (str(os_stats1)) utils.log_msg(msg, 'CHK-Stats') return os_stats1
def add_server_to_lb_network(cmgr, on_network_id, security_group_name_or_id, sid_range, keypair_name=None, username='******', password='******', flavor_id=1, image_id=None, image_name=None, public_network_id=None, extra_timeout=10, build_timeout=900, build_interval=5.0, halt=False): if halt: import pdb; pdb.set_trace() lb_network = cmgr.qsvc('net-show', on_network_id) lb_network_name = lb_network.get('name', None) sg = cmgr.qsvc('security-group-show', security_group_name_or_id) if keypair_name is None: # ssh keypair keypair = make_ssh_keypair(cmgr, lb_network_name) keypair_name = keypair['name'] else: keypair = None servers = {} for sid in sid_range: server_name = "%s-%s" % (lb_network_name, sid) server = NET.create_server_on_network( cmgr, on_network_id, security_group_name_or_id=sg['id'], key_name=keypair_name, server_name=server_name, image_id=image_id, image_name=image_name, flavor_id=flavor_id, wait_on_boot=False) servers[server['id']] = dict(server=server, fip=None) # servers need in status=ACTIVE before assign floatingip to them # vm1 should be active by now for server_id in servers.keys(): waiters.wait_for_server_status( cmgr.manager.servers_client, server_id, 'ACTIVE', extra_timeout=extra_timeout) for server_id, server in servers.items(): server['fip'] = NET.create_floatingip_for_server(cmgr, server_id, public_network_id) server['server'] = cmgr.nova('server-show', server_id) img_id = server['server']['image']['id'] img_name = cmgr.nova('image-show', img_id).get('name', '') server['image_name'] = img_name # wait for servers having their floating-ip (CH+newton takes long time) timeout = extra_timeout + build_timeout t0 = time.time() for server_id, server in servers.items(): while True: elapse_time = time.time() - t0 sv = cmgr.nova('server-show', server_id) if get_server_ip_address(sv, 'floating', lb_network_name): msg = ("Take %d seconds to assign floating-ip to server[%s]" % (int(elapse_time), sv.get('name'))) U.log_msg(msg, 'OS-LBaaS') break if elapse_time > timeout: raise Exception("Server did not get its floating-ip.") time.sleep(build_interval) lb_env = dict( username=username, password=password, keypair_name=keypair_name, keypair=keypair, security_group=sg, servers=servers) return lb_env