def retry_action_detail(action, retry_count=0, retry_interval=0.1): """ :param action: :param retry_count: :param retry_interval: :return: """ if retry_interval <= 0: logger_utils.fail('retry_interval <= 0 is not allowed, was: %s' % retry_interval) if retry_count < 0: logger_utils.fail('retry_count < 0 is not allowed, was %s' % retry_count) retry_count = int(retry_count) tries = 1 success, dbg_str = action() if success: return dbg_str for x in xrange(1, retry_count): tries += 1 success, dbg_str = action() if success: return dbg_str else: logger_utils.asleep(delay=retry_interval) logger.trace('Last Retry result: %s' % dbg_str) logger_utils.fail('%s, failed after %s tries' % (dbg_str, tries))
def verify_placement_vs_properties(vs_name): """ :param vs_name: :return: """ if not placement_get_vs_by_name(vs_name): return True vip1 = placement_get_vs_vip(vs_name) vip2 = vs_get_vip(vs_name) if vip1 != vip2: logger.trace('VS %s VIP %s and ' 'RM VIP %s dont match' % (vs_name, vip2, vip1)) logger_utils.fail('VS %s VIP %s and ' 'RM VIP %s dont match' % (vs_name, vip2, vip1)) se_grp1 = placement_get_vs_se_grp(vs_name) se_grp2 = vs_get_se_grp(vs_name) if se_grp1 != se_grp2: logger.trace('VS %s SE Grp %s and ' 'RM SE Grp %s dont match' % (vs_name, se_grp2, se_grp1)) logger_utils.fail('VS %s SE Grp %s and ' 'RM SE Grp %s dont match' % ( vs_name, se_grp2, se_grp1))
def validate_vs_dns_info(vs_name, retries=5, **kwargs): """ :param vs_name: :param retries: :param kwargs: :return: """ dns_vs_vip = kwargs.get('dns_vs_vip', '') if not dns_vs_vip: logger.info( "[SKIPPING] DNS check for VS as no DNS vip provided. Note, " "controller based DNS is not supported anymore.") return True import lib.vs_lib as vs_lib vs_json = vs_lib.get_vs(vs_name, tenant=kwargs.get('tenant', 'admin')) if vs_json['type'] == 'VS_TYPE_VH_CHILD': if rest.get_cloud_type() != 'openshift': logger.info("[SKIPPING] DNS check for VS as SNI child are not " "currently supported for non-openshift clouds") return True parent_ref = vs_json['vh_parent_vs_ref'] parent_uuid = parent_ref.split('/')[-1] _, parent_vs = rest.get('virtualservice', uuid=parent_uuid) parent_vs_name = parent_vs['name'] logger.info('SNI child VS detected; doing DNS on parent VS %s' % parent_vs_name) child_fqdn = vs_json['vh_domain_name'] parent_fqdns = [t['fqdn'] for t in parent_vs['dns_info']] if child_fqdn not in parent_fqdns: return False vs_name = parent_vs_name # REVIEW should it be parent or child name? vs_json = parent_vs dns_name = child_fqdn else: dns_name = vs_json['ipam_dns_records'][0]['fqdn'] logger.trace('vs_json: %s' % vs_json) if 'floating_ip' in vs_json: ip = vs_json['vip'][0]['floating_ip']['addr'] else: ip = vs_json['vip'][0]['ip_address']['addr'] ports = sorted([srv['port'] for srv in vs_json['services']]) logger.info("VS [%s]: IP %s, DNS %s, Ports: %s" % (vs_name, ip, dns_name, ports)) count = retries while count: if vs_lib.vs_check_ip_ports(vs_name, ip, dns_name, ports, dns_vs_vip=dns_vs_vip): return True count -= 1 logger_utils.asleep(delay=5) logger_utils.fail("DNS check failed!!")
def get_ip_mac_addr(ip, vm, retry): """ This function is to get the mac address for a given ip within the subnet. We have this function because our router is responding our arp request really slowwww.... Args: :param ip: IP Address for which you want to find MAC Address :type ip: str :param vm: VM Object on which you want to check :type vm: Object :param retry: Number of retrys :type retry: int Return: MAC Address for Given IP address """ retry_sec = retry cmd = 'rm /tmp/ping_out' vm.execute_command(cmd) if type(ipaddress.ip_address(unicode(ip))) == ipaddress.IPv6Address: cmd = 'ping6 -c 10 ' + ip + ' &> /tmp/ping_out &' else: cmd = 'ping -c 10 ' + ip + ' &> /tmp/ping_out &' logger.info(cmd) vm.execute_command(cmd) cmd = 'cat /tmp/ping_out' # retry until ping is successful. while retry_sec > 0: last_line = '' vm.execute_command(cmd) out = vm.execute_command('cat /tmp/ping_out') if out: last_line = out[-1] logger.trace('last line is: %s' % last_line) if 'rtt' in last_line: break sleep(1) retry_sec -= 1 if retry_sec < 1: fail( 'not able to connect to next hop! Please check by pinging gateway' ) # trying to get mac address from arp table if ":" in ip: cmd = 'ip -6 neigh | grep "' + ip + ' " | awk \'{print $5}\'' else: cmd = 'arp -n | grep "' + ip + ' " | awk \'{print $3}\'' out = vm.execute_command(cmd) if out == '': fail('Still doesn\'t get Arp entry! Please check by pinging gateway') return out[0][:-1]
def get_name_from_ref(ref): if not ref: return '' found = re.match("http.*api(.*)", ref) if not found: fail("ERROR! Cannot get api from ref") obj_type = found.group(1) logger.trace('get_name_from_ref: ref %s found %s obj_type %s' % (ref, str(found), obj_type)) status_code, json_data = rest.get(obj_type) return json_data.get('name')
def verify_placement_vs_resources(vs_name): """ :param vs_name: :return: """ if not placement_get_vs_by_name(vs_name): return True n1 = placement_get_vs_se_req(vs_name) n2 = vs_get_num_se_req(vs_name) if n1 != n2: logger.trace('VS SE Req %d and ' 'RM SE Req %d dont match' % (n2, n1)) logger_utils.fail('VS SE Req %d and ' 'RM SE Req %d dont match' % (n2, n1)) n1 = placement_get_vs_se_used(vs_name) n2 = vs_get_num_se_used(vs_name) if n1 != n2: logger.trace('VS SE Used %d and ' 'RM SE Used %d dont match' % (n2, n1)) logger_utils.fail('VS SE Used %d and ' 'RM SE Used %d dont match' % (n2, n1)) se_list1 = placement_get_vs_se_list(vs_name) se_list2 = vs_get_se_list(vs_name) # Set symmetric difference if len(set(se_list1) ^ set(se_list2)) > 0: logger.trace('VS %s SE List %s and ' 'RM SE List %s dont match' % (vs_name, se_list2, se_list1)) logger_utils.fail('VS %s SE Used %s and ' 'RM SE Used %s dont match' % ( vs_name, se_list2, se_list1)) se_list1 = placement_get_vs_se_list(vs_name, secondary=True) se_list2 = vs_get_se_list(vs_name, secondary=True) if len(set(se_list1) ^ set(se_list2)) > 0: logger.trace('VS %s Sec SE List %s and ' 'RM Sec SE List %s dont match' % (vs_name, se_list2, se_list1)) logger_utils.fail('VS %s Sec SE List %s and ' 'RM Sec SE List %s dont match' % ( vs_name, se_list2, se_list1)) se_list1 = placement_get_vs_se_list(vs_name, standby=True) se_list2 = vs_get_se_list(vs_name, standby=True) if len(set(se_list1) ^ set(se_list2)) > 0: logger.trace('VS %s Stby SE List %s and ' 'RM Stby SE List %s dont match' % ( vs_name, se_list2, se_list1)) logger_utils.fail('VS %s Stby SE List %s and ' 'RM Stby SE List %s dont match' % ( vs_name, se_list2, se_list1))
def add_ipam_and_dns_to_cloud(ipam_name=None, dns_name=None, **kwargs): cloud_name = kwargs.get('cloud', infra_utils.get_mode()['cloud']) cloud_obj = rest.ApiNode('cloud', name=cloud_name) status_code, json_data = cloud_obj.get() if ipam_name: json_data[ 'ipam_provider_ref'] = '/api/ipamdnsproviderprofile?name=%s' % ipam_name if dns_name: json_data[ 'dns_provider_ref'] = '/api/ipamdnsproviderprofile?name=%s' % dns_name logger.trace('JSON data: %s' % json_data) cloud_obj.put(data=json.dumps(json_data))
def get_name_by_uuid(obj_type, obj_uuid, **kwargs): """ Given object type and UUID, return name of the object :param obj_type: :param obj_uuid: :param kwargs: :return: """ resp_code, data = rest.get(obj_type, uuid=obj_uuid) name = data.get('name', None) logger.trace('URI: %s/%s Name: %s' % (obj_type, obj_uuid, name)) return name
def validate_after_delete(obj_type, obj_name, **kwargs): """ :param obj_type: :param obj_name: :param kwargs: :return: """ try: status_code, resp = rest.get(obj_type, name=obj_name) except: return logger.trace('Status: %s, Resp: %s' % (status_code, resp)) fail('%s deletion failed for object - %s' % (obj_type, obj_name))
def validate_after_delete(obj_type, obj_name, **kwargs): """ :param obj_type: :param obj_name: :param kwargs: :return: """ try: status_code, resp = rest.get(obj_type, name=obj_name) logger.trace('Status: %s, Resp: %s' % (status_code, resp)) logger_utils.fail('%s deletion failed for object - %s' % (obj_type, obj_name)) except: logger.info("%s %s successfully deleted" % (obj_type, obj_name))
def delete_instance(self): instance_name = self.vm_json.get('name') logger.trace("deleting instance %s" % instance_name) self.terminate_instance(instance_name)
def start_http_traffic(client_range, vs_name, vport, method, stop_on_failure, concurrent_conn_per_client=1, uri='/', **kwargs): """ Starts http traffic on range of clients specified in client_range Arguments: client_range Number of clients for the the traffic generation vs_name Virtualservice to balance the traffic between backend servers vport Listener port on virtual service stop_on_failure Flag to exit on first IO failure. Set it to 1 if you need concurrent_conn_per_client Concurrent connections per client uri URI of page to be requested exact state of the system while failure kwargs parameters passed to requests and validations need to be done """ skip_exc = kwargs.pop('skip_exception', 0) think_time = kwargs.pop('think_time', None) status_code = int(kwargs.pop('status_code', 200)) no_of_req_per_conn = int(kwargs.pop('no_of_req_per_conn', 1)) set_cookie_session = _bool_value(kwargs.pop('cookie_session', 'False')) set_cookies_per_request = _bool_value( kwargs.pop('cookies_per_request', 'False')) to_floating_vip = kwargs.pop('to_floating_vip', None) random_urls = int(kwargs.pop('random_urls', 0)) random_urls_repeat = int(kwargs.pop('random_urls_repeat', 0)) if random_urls: if not random_urls_repeat: random_urls_repeat = 1 uri = [] for i in range(random_urls): for j in range(random_urls_repeat): uri.append('/randomurl' + str(i)) urls = get_url(vs_name, vport, uri, skip_exc=skip_exc, vip_id=kwargs.pop('vip_id', '0'), to_floating_vip=to_floating_vip) client_vm, client_ips = traffic_manager.get_client_handles( client_range, concurrent_conn_per_client) if to_floating_vip: vip = vs_lib.get_floating_vip(vs_name) else: vip = vs_lib.get_vip(vs_name) out = client_vm.execute_command('arp -a | grep %s' % vip) logger.debug('Arptable on client: %s' % out) req_params, validations, prints = parse_input(urls, status_code, **kwargs) time_stamp = time.strftime("%Y%m%d%H%M%S") + str(time.clock()) logger.info('Start traffic at' + str(datetime.now())) log_file = '/tmp/httptest_io_error_' + time_stamp + client_range + '.log' cmd = '/root/client/tools/httptest.py ' cmd += '--urls \'%s\' ' % '\' \''.join(urls) cmd += '--clients %s ' % ' '.join(client_ips) cmd += '--stop-on-failure %s ' % stop_on_failure cmd += '--log-file %s ' % log_file cmd += '--method %s ' % method cmd += '--concurrent 1 ' cmd += '--requests %s ' % no_of_req_per_conn cmd += '--req-params \'%s\' ' % req_params cmd += '--validations \'%s\' ' % validations cmd += '--prints \'%s\' ' % prints if set_cookie_session: cmd += '--cookie-session ' if set_cookies_per_request: cmd += '--cookies-per-request ' if think_time: cmd += '--think-time %s ' % think_time cmd += '&> /tmp/httptest &' logger.trace(cmd) #out = client_vm.execute_command("rm -rf /tmp/httptest_traffic_check ", noerr=True) out = client_vm.execute_command(cmd) # Started Http Traffic check #search_str = "Started Http Traffic on client" #res = start_traffic_check("/tmp/httptest_traffic_check", search_str) #if res: # logger.error("HTTP Traffic is not started something went wrong") # reason = client_vm.execute_command(" tail -5 %s" % log_file) # logger.error(" %s" % (reason)) # reason = client_vm.execute_command(" tail -5 /tmp/httptest ") # logger.error(" %s" % (reason)) # raise RuntimeError("HTTP Request Traffic not started") #logger.info("HTTP Traffic is running, responses are logged in file: /tmp/httptest_resp_summary_time_stamp") logger.debug('End traffic at %s' % str(datetime.now())) logger.trace('out: %s' % out) return log_file
def pool_wellness_check(pool_name, t_state, t_num, t_up, t_enabled, skip_detail_check=0): """ :param pool_name: :param t_state: :param t_num: :param t_up: :param t_enabled: :param skip_detail_check: :return: """ logger_utils.asleep(msg='wait', delay=10) t_num = int(t_num) t_up = int(t_up) t_enabled = int(t_enabled) logger.info( '## start pool wellness check pool=%s t_state=%s t_num=%d t_up=%d t_enabled=%d' % (pool_name, t_state, t_num, t_up, t_enabled)) pool, pool_summary, pool_detail = cache_pool(pool_name) try: summary_oper_state = pool_summary['oper_status']['state'] if isinstance(pool_detail, list): detail_oper_state = pool_detail[0]['oper_status']['state'] pool_det = pool_detail[0] else: detail_oper_state = pool_detail['oper_status']['state'] pool_det = pool_detail except KeyError as err_msg: logger_utils.fail('## oper status not available: %s' % err_msg) if not skip_detail_check: if summary_oper_state != detail_oper_state: logger_utils.error( "Cache issue - summary state %s != detail state %s" % (summary_oper_state, detail_oper_state)) logger_utils.fail( "Cache issue - summary state %s != detail state %s" % (summary_oper_state, detail_oper_state)) if summary_oper_state != t_state: logger.debug("summary state %s != expected state %s" % (summary_oper_state, t_state)) logger_utils.fail("summary state %s != expected state %s" % (summary_oper_state, t_state)) if int(pool_summary['num_servers']) != t_num: logger.trace( "num servers mismatch (e-%d, s-%d, d-%d)" % (t_num, pool_summary['num_servers'], pool_det['num_servers'])) logger.fail( "num servers mismatch (e-%d, s-%d, d-%d)" % (t_num, pool_summary['num_servers'], pool_det['num_servers'])) if int(pool_summary['num_servers_enabled']) != t_enabled: logger.trace("num servers enabled mismatch (e-%d, s-%d, d-%d)" % (t_enabled, pool_summary['num_servers_enabled'], pool_det['num_servers_enabled'])) logger_utils.fail("num servers enabled mismatch (e-%d, s-%d, d-%d)" % (t_enabled, pool_summary['num_servers_enabled'], pool_det['num_servers_enabled'])) if int(pool_summary['num_servers_up']) != t_up: logger.trace( "num servers up mismatch (e-%d, s-%d, d-%d)" % (t_up, pool_summary['num_servers_up'], pool_det['num_servers_up'])) logger_utils.fail( "num servers up mismatch (e-%d, s-%d, d-%d)" % (t_up, pool_summary['num_servers_up'], pool_det['num_servers_up'])) return True
def validate_after_create(obj_type, obj_name): url = '%s/%s' % (obj_type, obj_name) logger.trace('Validate_after_create: %s' % str(obj_name)) rest.get(url)
def config_consistency_check(tenant='admin'): resp_code, resp = rest.get('configconsistencycheck') consistent = resp['consistent'] if not consistent: logger.trace('config_consistency_check failed, error:%s' % resp) logger_utils.fail('config_consistency_check failed, error:%s' % resp)