def is_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0):
for now in utils.run_till_timeout(time_out, ping_timeout):
reachable = dest_is_reachable(ssh_client, dest_ip)
if reachable:
return True
mesg = ("Dest[%s] not-reachable retry in %s seconds."
% (dest_ip, time_out))
utils.log_msg(mesg)
return False
def run_l7_switching(on_server_name_list, lb_vip_address, url_path='',
count=4):
resp_urls = lbaas2.count_http_servers(lb_vip_address,
count=count,
url_path=url_path)
if not url_responses_are_OK(resp_urls, on_server_name_list):
utils.log_msg("http://%s/%s redirected to wrong pool" % (
lb_vip_address, url_path), "LBaaS-L7 ERROR")
return False
return True
def isnot_reachable(ssh_client, dest_ip, time_out=60.0, ping_timeout=5.0,
idle_time=2.0):
if idle_time > 0.0:
time.sleep(idle_time)
for now in utils.run_till_timeout(time_out, ping_timeout):
reachable = dest_is_reachable(ssh_client, dest_ip)
if not reachable:
return True
mesg = ("Dest[%s] is reachable retry in %s seconds."
% (dest_ip, time_out))
utils.log_msg(mesg)
return False
def dest_is_reachable(ssh_client, dest_ip):
XPTN = r"(\d+).*transmit.*(\d+).*receive.*(\d+).*loss"
try:
result = ssh_client.ping_host(dest_ip)
utils.log_msg(result)
m = re.search(XPTN, result, (re.I | re.M))
if m and int(m.group(1)) > 0 and int(m.group(3)) == 0:
return True
else:
return False
except Exception:
tb_str = traceback.format_exc()
mesg = ("ERROR test dest_ip[%s] is reachable:\n%s" % (
dest_ip, tb_str))
utils.log_msg(mesg)
return False
def delete_the_loadbalancer(cmgr, lb_id, waitfor_delete=800,
pause_before_wait=2.0, interval=5.0):
cmgr.lbaas("loadbalancer-delete", lb_id)
time_start = time.time()
end_time = time.time() + waitfor_delete
time.sleep(pause_before_wait)
while (time.time() < end_time):
lb_list = cmgr.lbaas("loadbalancer-list", id=lb_id)
if len(lb_list) == 0:
break
time.sleep(interval)
etime = time.time() - time_start
utils.log_msg("elapse-time,delete-loadbalancer[%s]" % etime,
"OS-Stats")
return cmgr.lbaas("loadbalancer-list", id=lb_id)
def create_solaris(adm, os_auth_url, halt=False,
password="******"):
if halt:
import pdb;
pdb.set_trace()
# get-or-create Sun solaris system's admin Sun
try:
tenant = utils.fgrep(adm.keys('tenant-list'), name=r'^Sun$')
if len(tenant) < 1:
Sun = icreds.create_admin_project('Sun', password)
sun = utils.get_mimic_manager_cli(os_auth_url, 'Sun', password)
except Exception:
tb_str = traceback.format_exc()
mesg = ("ERROR creating/retriving Admin user[%s]:\n%s" % (
'Sun', tb_str))
utils.log_msg(mesg)
# our solar system has 8 planets'
sun_planets = ['Mercury', 'Venus', 'Earth', 'Mars',
'Jupiter', 'Satun', 'Uranus', 'Neptune']
dwarf_planets = ["Haumea", "Eris", "Ceres", "Pluto", "Makemake"]
tenants = {}
for planet in sun_planets + dwarf_planets + ["Moon"]:
tenant = utils.fgrep(adm.keys('tenant-list'), name=planet)
if len(tenant) < 1:
# tenant not exist, create it; default password=itempest
tenant = icreds.create_primary_project(planet, password)
try:
tenant = adm.keys('tenant_get_by_name', planet)
tenants[planet] = tenant
# by default tenant can only have instances=10
adm.nova('quota-update', tenant['id'],
instances=tenant_max_instances)
adm.qsvc('quota-incr-by', tenant['id'], 2)
except Exception:
pass
else:
tenants[planet] = tenant[0]
return tenants
def add_floatingip_to_server_and_test_reachibility(
cli_mgr, server_id, security_group_id=None, action='add',
check_interval=5, check_duration=60):
mesg_p = "%s reach server[%s %s] with-ip[%s %s]."
try:
ss = cli_mgr.nova('server-show', server_id)
fip_result = add_floatingip_to_server(
cli_mgr, server_id,
security_group_id=security_group_id,
check_interval=check_interval,
check_duration=check_duration,
action=action)
m_type = "CAN" if fip_result[1] else "CAN'T"
mesg = mesg_p % (m_type, server_id, ss['name'],
fip_result[0]['fixed_ip_address'],
fip_result[0]['floating_ip_address'])
utils.log_msg(mesg)
return fip_result
except Exception:
tb_str = traceback.format_exc()
mesg = ("ERROR creating floatingip for server[%s]:\n%s" % (
server_id, tb_str))
utils.log_msg(mesg)
return (server_id, False)
def poke_http_stats_change(nsxt_client, cmgr, lb2_name,
interval=5.0, poke_count=100,
no_http_traffic=False):
lb2_fip = lbaas2.get_loadbalancer_floatingip(cmgr, lb2_name)
lb2_web_ip = lb2_fip[1]['floating_ip_address']
lb = cmgr.lbaas('loadbalancer-show', lb2_name)
lb_port = cmgr.qsvc('port-show', lb['vip_port_id'])
os_security_group_id = lb_port['security_groups'][0]
_sgroup = cmgr.qsvc('security-group-show', os_security_group_id)
msg = "os-security-group-id = %s" % (os_security_group_id)
utils.log_msg(msg, 'CHK-Stats')
sg_rules = _sgroup['security_group_rules']
http_rule = [x for x in sg_rules if x['port_range_min'] >= 80][0]
filters = {'os-neutron-secgr-id': os_security_group_id}
fw_list = nsxt_client.list_firewall_sections(**filters)
sect_id = fw_list[0].get('id')
fw_rules = nsxt_client.get_firewall_section_rules(sect_id)
rule = [x for x in fw_rules if x['display_name'] == http_rule.get('id')]
rule_id = rule[0].get('id')
msg = "NSX STATS is at section[%s] rule_id[%s]" % (sect_id, rule_id)
utils.log_msg(msg, 'CHK-Stats')
nsx_stats = nsxt_client.get_firewall_section_rule_stats(sect_id, rule_id)
os_stats1 = log_lb2_stats(cmgr, lb2_name)
if no_http_traffic:
utils.log_msg("No http traffic initiated", 'CHK-Stats')
else:
lbaas2.count_http_servers(lb2_web_ip)
m_stats_change(nsxt_client, sect_id, rule_id, nsx_stats,
cmgr, lb2_name,
interval=interval, poke_count=poke_count)
os_stats2 = log_lb2_stats(cmgr, lb2_name)
dd = {}
for k in os_stats2.keys():
dd[k] = int(os_stats2[k]) - int(os_stats1[k])
utils.log_msg("%s" % str(dd), "OS-STATS-DIFF")
def m_stats_change(nsxt_client, sect_id, rule_id, nsx_stats,
cmgr, lb2_name,
interval=2.5, poke_count=150):
t0 = time.time()
for cnt in range(poke_count):
log_lb2_stats(cmgr, lb2_name)
ss = nsxt_client.get_firewall_section_rule_stats(sect_id, rule_id)
ss.pop('_schema', None)
ss.pop('rule_id', None)
msg = "#%02d %s" % (cnt, str(ss))
utils.log_msg(msg, 'CHK-Stats')
if ss.get('session_count') != nsx_stats.get('session_count'):
e_time = (time.time() - t0)
msg = "Take %d seconds for STATS to be updated" % int(e_time)
utils.log_msg(msg, 'CHK-Stats')
dd = {}
for k in ss.keys():
dd[k] = int(ss[k]) - int(nsx_stats[k])
utils.log_msg("%s" % str(dd), "NSX-STATS-DIFF")
return e_time
time.sleep(interval)
tt = (time.time() - t0)
raise Exception("After %d seconds, stats do not change!" % int(tt))
def show_toplogy(cli_mgr, return_topo=False, prefix=None,
router_id=None, delete_resources=False):
tenant_name = cli_mgr.manager.credentials.tenant_name
FMT_ROUTER = "%s>> {router_type} router: {name} {id}" % (' ' * 2)
FMT_ROUTER_O = "%s>> router: {name} {id}" % (' ' * 2)
FMT_X_GW1 = "%sGW: snat_enabled: {enable_snat}" % (' ' * 5)
FMT_X_GW2 = "%sfixed_ip: {external_fixed_ips}" % (' ' * (5 + 4))
FMT_X_ROUT = "%sroutes: {routes}" % (' ' * (5 + 4))
FMT_INTERFACE = "%s>> interface: {name} {id}" % (' ' * 6)
# FMT_SUBNETS = "%s subnets: {subnets}" % (' ' * 8)
FMT_SNET_ADDR = "%s subnet: {id} {name} cidr={cidr} gw={gateway_ip}" % (
' ' * 8)
FMT_SERVER = "%s>> server: {name} {id}" % (' ' * 10)
FMT_SERV_ADDR = "%s>> network: %s "
topo = []
topo_line = ["\nNetwork topology of tenant[%s]" % tenant_name]
s_list = cli_mgr.nova('server-list-with-detail')
if router_id:
router_list = cli_mgr.qsvc('router-list', id=router_id)
else:
router_list = cli_mgr.qsvc('router-list')
if prefix:
ser_name = "^%s" % prefix
s_list = utils.fgrep(s_list, name=ser_name)
router_list = utils.fgrep(router_list, name=ser_name)
sorted(router_list, key=itemgetter('name'))
for router in router_list:
rtr = _g_by_attr(router,
('id', 'name', 'router_type', 'distributed'))
rtr['_networks'] = []
if 'distributed' in router and router['distributed']:
rtr['router_type'] = 'distributed'
try:
topo_line.append(FMT_ROUTER.format(**rtr))
except:
topo_line.append(FMT_ROUTER_O.format(**rtr))
if type(router['external_gateway_info']) is dict:
xnet_info = router['external_gateway_info']
rtr['gateway'] = xnet_info
topo_line.append(FMT_X_GW1.format(**xnet_info))
try:
topo_line.append(FMT_X_GW2.format(**xnet_info))
except:
utils.log_msg("GW does not have external_fixed_ips: %s"
% xnet_info)
rtr['routes'] = router['routes']
if (type(router['routes']) in [list, tuple] and
len(router['routes']) > 0):
topo_line.append(FMT_X_ROUT.format(**router))
rp_list = cli_mgr.qsvc('router-port-list', router['id'])
for rp in rp_list:
network = cli_mgr.qsvc('net-show', rp['network_id'])
netwk = _g_by_attr(network, ('name', 'id'))
subnet_list = network['subnets']
netwk['port_id'] = rp['id']
netwk['_servers'] = []
topo_line.append(FMT_INTERFACE.format(**netwk))
netwk['subnets'] = []
for subnet_id in subnet_list:
subnet = cli_mgr.qsvc('subnet-show', subnet_id)
subnet = _g_by_attr(subnet, ('id', 'cidr', 'gateway_ip',
'allocation_pools', 'name'))
topo_line.append(FMT_SNET_ADDR.format(**subnet))
netwk['subnets'].append(subnet)
if_name = network['name']
if_servers = [s for s in s_list if if_name in s['addresses']]
for s in if_servers:
addr_dict = mdata.get_server_address(s)
no_if = len(addr_dict)
serv = _g_by_attr(s, ('name', 'id'))
serv['#interface'] = no_if
topo_line.append(
FMT_SERVER.format(**s) + " #interface=%s" % no_if)
if if_name in addr_dict:
serv['interface'] = addr_dict[if_name]
topo_line.append(
FMT_SERV_ADDR % (' ' * 14, addr_dict[if_name]))
netwk['_servers'].append(serv)
rtr['_networks'].append(netwk)
topo.append(rtr)
print("\n".join(topo_line))
if delete_resources:
_delete_topology(cli_mgr, topo)
return topo if return_topo else {}
def log_lb2_stats(cmgr, lb2_name):
os_stats1 = cmgr.lbaas('loadbalancer-stats', lb2_name)
msg = "OS-LB2-Stats %s" % (str(os_stats1))
utils.log_msg(msg, 'CHK-Stats')
return os_stats1
def add_server_to_lb_network(cmgr, on_network_id, security_group_name_or_id,
sid_range, keypair_name=None,
username='******', password='******',
flavor_id=1, image_id=None, image_name=None,
public_network_id=None, extra_timeout=10,
build_timeout=900, build_interval=5.0,
halt=False):
if halt:
import pdb;
pdb.set_trace()
lb_network = cmgr.qsvc('net-show', on_network_id)
lb_network_name = lb_network.get('name', None)
sg = cmgr.qsvc('security-group-show', security_group_name_or_id)
if keypair_name is None:
# ssh keypair
keypair = make_ssh_keypair(cmgr, lb_network_name)
keypair_name = keypair['name']
else:
keypair = None
servers = {}
for sid in sid_range:
server_name = "%s-%s" % (lb_network_name, sid)
server = NET.create_server_on_network(
cmgr, on_network_id, security_group_name_or_id=sg['id'],
key_name=keypair_name, server_name=server_name,
image_id=image_id, image_name=image_name,
flavor_id=flavor_id, wait_on_boot=False)
servers[server['id']] = dict(server=server, fip=None)
# servers need in status=ACTIVE before assign floatingip to them
# vm1 should be active by now
for server_id in servers.keys():
waiters.wait_for_server_status(
cmgr.manager.servers_client, server_id, 'ACTIVE',
extra_timeout=extra_timeout)
for server_id, server in servers.items():
server['fip'] = NET.create_floatingip_for_server(cmgr, server_id,
public_network_id)
server['server'] = cmgr.nova('server-show', server_id)
img_id = server['server']['image']['id']
img_name = cmgr.nova('image-show', img_id).get('name', '')
server['image_name'] = img_name
# wait for servers having their floating-ip (CH+newton takes long time)
timeout = extra_timeout + build_timeout
t0 = time.time()
for server_id, server in servers.items():
while True:
elapse_time = time.time() - t0
sv = cmgr.nova('server-show', server_id)
if get_server_ip_address(sv, 'floating', lb_network_name):
msg = ("Take %d seconds to assign floating-ip to server[%s]"
% (int(elapse_time), sv.get('name')))
U.log_msg(msg, 'OS-LBaaS')
break
if elapse_time > timeout:
raise Exception("Server did not get its floating-ip.")
time.sleep(build_interval)
lb_env = dict(
username=username, password=password,
keypair_name=keypair_name, keypair=keypair,
security_group=sg, servers=servers)
return lb_env