def retry_action_detail(action, retry_count=0, retry_interval=0.1):
"""
:param action:
:param retry_count:
:param retry_interval:
:return:
"""
if retry_interval <= 0:
logger_utils.fail('retry_interval <= 0 is not allowed, was: %s' %
retry_interval)
if retry_count < 0:
logger_utils.fail('retry_count < 0 is not allowed, was %s' %
retry_count)
retry_count = int(retry_count)
tries = 1
success, dbg_str = action()
if success:
return dbg_str
for x in xrange(1, retry_count):
tries += 1
success, dbg_str = action()
if success:
return dbg_str
else:
logger_utils.asleep(delay=retry_interval)
logger.trace('Last Retry result: %s' % dbg_str)
logger_utils.fail('%s, failed after %s tries' % (dbg_str, tries))
def verify_placement_vs_properties(vs_name):
"""
:param vs_name:
:return:
"""
if not placement_get_vs_by_name(vs_name):
return True
vip1 = placement_get_vs_vip(vs_name)
vip2 = vs_get_vip(vs_name)
if vip1 != vip2:
logger.trace('VS %s VIP %s and '
'RM VIP %s dont match' % (vs_name, vip2, vip1))
logger_utils.fail('VS %s VIP %s and '
'RM VIP %s dont match' % (vs_name, vip2, vip1))
se_grp1 = placement_get_vs_se_grp(vs_name)
se_grp2 = vs_get_se_grp(vs_name)
if se_grp1 != se_grp2:
logger.trace('VS %s SE Grp %s and '
'RM SE Grp %s dont match' % (vs_name, se_grp2, se_grp1))
logger_utils.fail('VS %s SE Grp %s and '
'RM SE Grp %s dont match' % (
vs_name, se_grp2, se_grp1))
def validate_vs_dns_info(vs_name, retries=5, **kwargs):
"""
:param vs_name:
:param retries:
:param kwargs:
:return:
"""
dns_vs_vip = kwargs.get('dns_vs_vip', '')
if not dns_vs_vip:
logger.info(
"[SKIPPING] DNS check for VS as no DNS vip provided. Note, "
"controller based DNS is not supported anymore.")
return True
import lib.vs_lib as vs_lib
vs_json = vs_lib.get_vs(vs_name, tenant=kwargs.get('tenant', 'admin'))
if vs_json['type'] == 'VS_TYPE_VH_CHILD':
if rest.get_cloud_type() != 'openshift':
logger.info("[SKIPPING] DNS check for VS as SNI child are not "
"currently supported for non-openshift clouds")
return True
parent_ref = vs_json['vh_parent_vs_ref']
parent_uuid = parent_ref.split('/')[-1]
_, parent_vs = rest.get('virtualservice', uuid=parent_uuid)
parent_vs_name = parent_vs['name']
logger.info('SNI child VS detected; doing DNS on parent VS %s' %
parent_vs_name)
child_fqdn = vs_json['vh_domain_name']
parent_fqdns = [t['fqdn'] for t in parent_vs['dns_info']]
if child_fqdn not in parent_fqdns:
return False
vs_name = parent_vs_name # REVIEW should it be parent or child name?
vs_json = parent_vs
dns_name = child_fqdn
else:
dns_name = vs_json['ipam_dns_records'][0]['fqdn']
logger.trace('vs_json: %s' % vs_json)
if 'floating_ip' in vs_json:
ip = vs_json['vip'][0]['floating_ip']['addr']
else:
ip = vs_json['vip'][0]['ip_address']['addr']
ports = sorted([srv['port'] for srv in vs_json['services']])
logger.info("VS [%s]: IP %s, DNS %s, Ports: %s" %
(vs_name, ip, dns_name, ports))
count = retries
while count:
if vs_lib.vs_check_ip_ports(vs_name,
ip,
dns_name,
ports,
dns_vs_vip=dns_vs_vip):
return True
count -= 1
logger_utils.asleep(delay=5)
logger_utils.fail("DNS check failed!!")
def get_ip_mac_addr(ip, vm, retry):
""" This function is to get the mac address for a given ip within
the subnet. We have this function because our router is responding
our arp request really slowwww....
Args:
:param ip: IP Address for which you want to find MAC Address
:type ip: str
:param vm: VM Object on which you want to check
:type vm: Object
:param retry: Number of retrys
:type retry: int
Return:
MAC Address for Given IP address
"""
retry_sec = retry
cmd = 'rm /tmp/ping_out'
vm.execute_command(cmd)
if type(ipaddress.ip_address(unicode(ip))) == ipaddress.IPv6Address:
cmd = 'ping6 -c 10 ' + ip + ' &> /tmp/ping_out &'
else:
cmd = 'ping -c 10 ' + ip + ' &> /tmp/ping_out &'
logger.info(cmd)
vm.execute_command(cmd)
cmd = 'cat /tmp/ping_out'
# retry until ping is successful.
while retry_sec > 0:
last_line = ''
vm.execute_command(cmd)
out = vm.execute_command('cat /tmp/ping_out')
if out:
last_line = out[-1]
logger.trace('last line is: %s' % last_line)
if 'rtt' in last_line:
break
sleep(1)
retry_sec -= 1
if retry_sec < 1:
fail(
'not able to connect to next hop! Please check by pinging gateway'
)
# trying to get mac address from arp table
if ":" in ip:
cmd = 'ip -6 neigh | grep "' + ip + ' " | awk \'{print $5}\''
else:
cmd = 'arp -n | grep "' + ip + ' " | awk \'{print $3}\''
out = vm.execute_command(cmd)
if out == '':
fail('Still doesn\'t get Arp entry! Please check by pinging gateway')
return out[0][:-1]
def get_name_from_ref(ref):
if not ref:
return ''
found = re.match("http.*api(.*)", ref)
if not found:
fail("ERROR! Cannot get api from ref")
obj_type = found.group(1)
logger.trace('get_name_from_ref: ref %s found %s obj_type %s' %
(ref, str(found), obj_type))
status_code, json_data = rest.get(obj_type)
return json_data.get('name')
def verify_placement_vs_resources(vs_name):
"""
:param vs_name:
:return:
"""
if not placement_get_vs_by_name(vs_name):
return True
n1 = placement_get_vs_se_req(vs_name)
n2 = vs_get_num_se_req(vs_name)
if n1 != n2:
logger.trace('VS SE Req %d and '
'RM SE Req %d dont match' % (n2, n1))
logger_utils.fail('VS SE Req %d and '
'RM SE Req %d dont match' % (n2, n1))
n1 = placement_get_vs_se_used(vs_name)
n2 = vs_get_num_se_used(vs_name)
if n1 != n2:
logger.trace('VS SE Used %d and '
'RM SE Used %d dont match' % (n2, n1))
logger_utils.fail('VS SE Used %d and '
'RM SE Used %d dont match' % (n2, n1))
se_list1 = placement_get_vs_se_list(vs_name)
se_list2 = vs_get_se_list(vs_name)
# Set symmetric difference
if len(set(se_list1) ^ set(se_list2)) > 0:
logger.trace('VS %s SE List %s and '
'RM SE List %s dont match' % (vs_name, se_list2, se_list1))
logger_utils.fail('VS %s SE Used %s and '
'RM SE Used %s dont match' % (
vs_name, se_list2, se_list1))
se_list1 = placement_get_vs_se_list(vs_name, secondary=True)
se_list2 = vs_get_se_list(vs_name, secondary=True)
if len(set(se_list1) ^ set(se_list2)) > 0:
logger.trace('VS %s Sec SE List %s and '
'RM Sec SE List %s dont match' % (vs_name, se_list2, se_list1))
logger_utils.fail('VS %s Sec SE List %s and '
'RM Sec SE List %s dont match' % (
vs_name, se_list2, se_list1))
se_list1 = placement_get_vs_se_list(vs_name, standby=True)
se_list2 = vs_get_se_list(vs_name, standby=True)
if len(set(se_list1) ^ set(se_list2)) > 0:
logger.trace('VS %s Stby SE List %s and '
'RM Stby SE List %s dont match' % (
vs_name, se_list2, se_list1))
logger_utils.fail('VS %s Stby SE List %s and '
'RM Stby SE List %s dont match' % (
vs_name, se_list2, se_list1))
def add_ipam_and_dns_to_cloud(ipam_name=None, dns_name=None, **kwargs):
cloud_name = kwargs.get('cloud', infra_utils.get_mode()['cloud'])
cloud_obj = rest.ApiNode('cloud', name=cloud_name)
status_code, json_data = cloud_obj.get()
if ipam_name:
json_data[
'ipam_provider_ref'] = '/api/ipamdnsproviderprofile?name=%s' % ipam_name
if dns_name:
json_data[
'dns_provider_ref'] = '/api/ipamdnsproviderprofile?name=%s' % dns_name
logger.trace('JSON data: %s' % json_data)
cloud_obj.put(data=json.dumps(json_data))
def get_name_by_uuid(obj_type, obj_uuid, **kwargs):
"""
Given object type and UUID, return name of the object
:param obj_type:
:param obj_uuid:
:param kwargs:
:return:
"""
resp_code, data = rest.get(obj_type, uuid=obj_uuid)
name = data.get('name', None)
logger.trace('URI: %s/%s Name: %s' % (obj_type, obj_uuid, name))
return name
def validate_after_delete(obj_type, obj_name, **kwargs):
"""
:param obj_type:
:param obj_name:
:param kwargs:
:return:
"""
try:
status_code, resp = rest.get(obj_type, name=obj_name)
except:
return
logger.trace('Status: %s, Resp: %s' % (status_code, resp))
fail('%s deletion failed for object - %s' % (obj_type, obj_name))
def validate_after_delete(obj_type, obj_name, **kwargs):
"""
:param obj_type:
:param obj_name:
:param kwargs:
:return:
"""
try:
status_code, resp = rest.get(obj_type, name=obj_name)
logger.trace('Status: %s, Resp: %s' % (status_code, resp))
logger_utils.fail('%s deletion failed for object - %s' %
(obj_type, obj_name))
except:
logger.info("%s %s successfully deleted" % (obj_type, obj_name))
def delete_instance(self):
instance_name = self.vm_json.get('name')
logger.trace("deleting instance %s" % instance_name)
self.terminate_instance(instance_name)
def start_http_traffic(client_range,
vs_name,
vport,
method,
stop_on_failure,
concurrent_conn_per_client=1,
uri='/',
**kwargs):
"""
Starts http traffic on range of clients specified in client_range
Arguments:
client_range Number of clients for the the traffic
generation
vs_name Virtualservice to balance the traffic
between backend servers
vport Listener port on virtual service
stop_on_failure Flag to exit on first IO failure.
Set it to 1 if you need
concurrent_conn_per_client Concurrent connections per client
uri URI of page to be requested
exact state of the system while failure
kwargs parameters passed to requests and
validations need to be
done
"""
skip_exc = kwargs.pop('skip_exception', 0)
think_time = kwargs.pop('think_time', None)
status_code = int(kwargs.pop('status_code', 200))
no_of_req_per_conn = int(kwargs.pop('no_of_req_per_conn', 1))
set_cookie_session = _bool_value(kwargs.pop('cookie_session', 'False'))
set_cookies_per_request = _bool_value(
kwargs.pop('cookies_per_request', 'False'))
to_floating_vip = kwargs.pop('to_floating_vip', None)
random_urls = int(kwargs.pop('random_urls', 0))
random_urls_repeat = int(kwargs.pop('random_urls_repeat', 0))
if random_urls:
if not random_urls_repeat:
random_urls_repeat = 1
uri = []
for i in range(random_urls):
for j in range(random_urls_repeat):
uri.append('/randomurl' + str(i))
urls = get_url(vs_name,
vport,
uri,
skip_exc=skip_exc,
vip_id=kwargs.pop('vip_id', '0'),
to_floating_vip=to_floating_vip)
client_vm, client_ips = traffic_manager.get_client_handles(
client_range, concurrent_conn_per_client)
if to_floating_vip:
vip = vs_lib.get_floating_vip(vs_name)
else:
vip = vs_lib.get_vip(vs_name)
out = client_vm.execute_command('arp -a | grep %s' % vip)
logger.debug('Arptable on client: %s' % out)
req_params, validations, prints = parse_input(urls, status_code, **kwargs)
time_stamp = time.strftime("%Y%m%d%H%M%S") + str(time.clock())
logger.info('Start traffic at' + str(datetime.now()))
log_file = '/tmp/httptest_io_error_' + time_stamp + client_range + '.log'
cmd = '/root/client/tools/httptest.py '
cmd += '--urls \'%s\' ' % '\' \''.join(urls)
cmd += '--clients %s ' % ' '.join(client_ips)
cmd += '--stop-on-failure %s ' % stop_on_failure
cmd += '--log-file %s ' % log_file
cmd += '--method %s ' % method
cmd += '--concurrent 1 '
cmd += '--requests %s ' % no_of_req_per_conn
cmd += '--req-params \'%s\' ' % req_params
cmd += '--validations \'%s\' ' % validations
cmd += '--prints \'%s\' ' % prints
if set_cookie_session:
cmd += '--cookie-session '
if set_cookies_per_request:
cmd += '--cookies-per-request '
if think_time:
cmd += '--think-time %s ' % think_time
cmd += '&> /tmp/httptest &'
logger.trace(cmd)
#out = client_vm.execute_command("rm -rf /tmp/httptest_traffic_check ", noerr=True)
out = client_vm.execute_command(cmd)
# Started Http Traffic check
#search_str = "Started Http Traffic on client"
#res = start_traffic_check("/tmp/httptest_traffic_check", search_str)
#if res:
# logger.error("HTTP Traffic is not started something went wrong")
# reason = client_vm.execute_command(" tail -5 %s" % log_file)
# logger.error(" %s" % (reason))
# reason = client_vm.execute_command(" tail -5 /tmp/httptest ")
# logger.error(" %s" % (reason))
# raise RuntimeError("HTTP Request Traffic not started")
#logger.info("HTTP Traffic is running, responses are logged in file: /tmp/httptest_resp_summary_time_stamp")
logger.debug('End traffic at %s' % str(datetime.now()))
logger.trace('out: %s' % out)
return log_file
def pool_wellness_check(pool_name,
t_state,
t_num,
t_up,
t_enabled,
skip_detail_check=0):
"""
:param pool_name:
:param t_state:
:param t_num:
:param t_up:
:param t_enabled:
:param skip_detail_check:
:return:
"""
logger_utils.asleep(msg='wait', delay=10)
t_num = int(t_num)
t_up = int(t_up)
t_enabled = int(t_enabled)
logger.info(
'## start pool wellness check pool=%s t_state=%s t_num=%d t_up=%d t_enabled=%d'
% (pool_name, t_state, t_num, t_up, t_enabled))
pool, pool_summary, pool_detail = cache_pool(pool_name)
try:
summary_oper_state = pool_summary['oper_status']['state']
if isinstance(pool_detail, list):
detail_oper_state = pool_detail[0]['oper_status']['state']
pool_det = pool_detail[0]
else:
detail_oper_state = pool_detail['oper_status']['state']
pool_det = pool_detail
except KeyError as err_msg:
logger_utils.fail('## oper status not available: %s' % err_msg)
if not skip_detail_check:
if summary_oper_state != detail_oper_state:
logger_utils.error(
"Cache issue - summary state %s != detail state %s" %
(summary_oper_state, detail_oper_state))
logger_utils.fail(
"Cache issue - summary state %s != detail state %s" %
(summary_oper_state, detail_oper_state))
if summary_oper_state != t_state:
logger.debug("summary state %s != expected state %s" %
(summary_oper_state, t_state))
logger_utils.fail("summary state %s != expected state %s" %
(summary_oper_state, t_state))
if int(pool_summary['num_servers']) != t_num:
logger.trace(
"num servers mismatch (e-%d, s-%d, d-%d)" %
(t_num, pool_summary['num_servers'], pool_det['num_servers']))
logger.fail(
"num servers mismatch (e-%d, s-%d, d-%d)" %
(t_num, pool_summary['num_servers'], pool_det['num_servers']))
if int(pool_summary['num_servers_enabled']) != t_enabled:
logger.trace("num servers enabled mismatch (e-%d, s-%d, d-%d)" %
(t_enabled, pool_summary['num_servers_enabled'],
pool_det['num_servers_enabled']))
logger_utils.fail("num servers enabled mismatch (e-%d, s-%d, d-%d)" %
(t_enabled, pool_summary['num_servers_enabled'],
pool_det['num_servers_enabled']))
if int(pool_summary['num_servers_up']) != t_up:
logger.trace(
"num servers up mismatch (e-%d, s-%d, d-%d)" %
(t_up, pool_summary['num_servers_up'], pool_det['num_servers_up']))
logger_utils.fail(
"num servers up mismatch (e-%d, s-%d, d-%d)" %
(t_up, pool_summary['num_servers_up'], pool_det['num_servers_up']))
return True
def validate_after_create(obj_type, obj_name):
url = '%s/%s' % (obj_type, obj_name)
logger.trace('Validate_after_create: %s' % str(obj_name))
rest.get(url)
def config_consistency_check(tenant='admin'):
resp_code, resp = rest.get('configconsistencycheck')
consistent = resp['consistent']
if not consistent:
logger.trace('config_consistency_check failed, error:%s' % resp)
logger_utils.fail('config_consistency_check failed, error:%s' % resp)
