def _reconnect(self, steps, timeout): '''Reconnect to the device if needed''' if self.process in self.reconnect: ha = self.abstract.sdk.libs.abstracted_libs.ha.HA( device=self.device) with steps.start( 'The device is reloading when restarting this process', continue_=True) as step: # Waiting 4-7 mins for switchover to complete log.info( "Waiting for a maximum of 7 minutes for device switchover to completess" ) check_time = Timeout(max_time=700, interval=20) while check_time.iterate(): check_time.sleep() continue self.device.destroy() time.sleep(30) temp = TempResult(container=step) while timeout.iterate(): try: self.device.connect(prompt_recovery=True) except Exception as e: temp.failed('Could not reconnect to the device', from_exception=e) timeout.sleep() continue temp.passed('Reconnected to the device') break temp.result() # check show module with steps.start('Check module status after reconnection', continue_=True) as step: temp = TempResult(container=step) while timeout.iterate(): try: ha.check_module() except AttributeError as e: temp.failed( 'Could not find mandatory information from show module', from_exception=e) continue except AssertionError as e: temp.failed('Modules are not ready', from_exception=e) timeout.sleep() continue temp.passed('Modules are ready') break temp.result()
def verify_tunnel_stats( device, tunnel_id, direction, tx_packets, pkt_rate, max_time=3, check_interval=5 ): """ Verify acl usage Args: device (`obj`): Device object tunnel_id (`str`): Tunnel id direction (`str`): Traffic direction tx_packets (`int`): Transmit packets pkt_rate (`str`): Frames per seconds max_time ('int',optional): Maximum wait time for the trigger, in second. Default: 3 check_interval (`int`, optional): Wait time between iterations when looping is needed, in second. Default: 5 Returns: True False """ out = None try: out = device.parse("show interfaces {intf} stats".format(intf=tunnel_id)) except SchemaEmptyParserError: pass timeout = Timeout(max_time, check_interval) while timeout.iterate(): if out: get_packet_count_in = out[tunnel_id]['switching_path']['distributed_cache']['pkts_in'] get_packet_count_out = out[tunnel_id]['switching_path']['distributed_cache']['pkts_out'] expected_max_pkts = tx_packets + int(pkt_rate) expected_min_pkts = tx_packets - int(pkt_rate) if direction == 'egress': if not get_packet_count_out > 0: return False if (get_packet_count_out >= expected_min_pkts) and (get_packet_count_out <= expected_max_pkts): return True if direction == 'ingress': if not get_packet_count_in > 0: return False if (get_packet_count_in >= expected_min_pkts) and (get_packet_count_in <= expected_max_pkts): return True timeout.sleep() return False
def verify_bgp_peer_as(device, peer_address, expected_peer_as, max_time=60, check_interval=10): """ Verify bgp peer AS number Args: device('obj'): device to use peer_address('str'): Peer interface expected_peer_as ('int'): Expected peer AS number max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds. check_interval ('int', optional): How often to check. Default to 10 seconds. Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) # show commands: "show bgp neighbor" # { # "bgp-information": { # "bgp-peer": [ # { # "local-address": "10.189.5.252", # "local-as": "65171", # "peer-address": "10.49.216.179", # "peer-as": "65171", <----------------------- while timeout.iterate(): try: out = device.parse('show bgp neighbor') except SchemaEmptyParserError: timeout.sleep() continue peers_list = out.q.get_values("bgp-peer") for peer in peers_list: addr = peer.get('peer-address') peer_as = Dq(peer).get_values('peer-as', 0) # 20.0.0.3+63208 addr = addr.split('+')[0] if addr == peer_address.split('/')[0] and int( peer_as) == expected_peer_as: return True timeout.sleep() return False
def verify_no_ospf_interface_in_database(device, expected_interface, max_time=60, check_interval=10): """ Verifies ospf interface exists with criteria Args: device ('obj'): device to use expected_interface ('str'): Interface to use max_time ('int', optional): Maximum time to keep checking. Defaults to 60 seconds. check_interval (`int`): Check interval, default: 10 Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): exist = False try: out = device.parse('show ospf database extensive') except SchemaEmptyParserError: timeout.sleep() continue #'ospf-database':[ # {'lsa-type': # 'Router', # 'lsa-id': '1.1.1.1' # } #] for ospf_database in Dq(out).get_values('ospf-database'): #'ospf-external-lsa': # {'address-mask': '255.255.255.255', # 'ospf-external-lsa-topology': { # 'ospf-topology-name': # 'default'}} ospf_external_lsa = ospf_database.get('ospf-external-lsa', {}) advertising_router = ospf_database.get('advertising-router', None) if expected_interface == advertising_router: exist = True if exist: timeout.sleep() continue else: return False return True
def verify_ping( device, address, expected_max_success_rate=100, expected_min_success_rate=0, count=None, source=None, max_time=60, check_interval=10, ): """Verify ping Args: device ('obj'): Device object address ('str'): Address value expected_max_success_rate (int): Expected maximum success rate expected_min_success_rate (int): Expected minimum success rate count ('int'): Count value for ping command source ('str'): Source IP address, default: None max_time (`int`): Max time, default: 30 check_interval (`int`): Check interval, default: 10 """ p = re.compile(r"Success +rate +is +(?P<rate>\d+) +percent.*") timeout = Timeout(max_time, check_interval) while timeout.iterate(): if address and count and source: cmd = 'ping {address} source {source} repeat {count}'.format( address=address, source=source, count=count) elif address and count: cmd = 'ping {address} repeat {count}'.format(address=address, count=count) elif address and source: cmd = 'ping {address} source {source}'.format(address=address, source=source) elif address: cmd = 'ping {address}'.format(address=address) else: log.info('Need to pass address as argument') return False try: out = device.execute(cmd) except SubCommandFailure as e: timeout.sleep() continue rate = int(p.search(out).groupdict().get('rate', 0)) if expected_max_success_rate >= rate >= expected_min_success_rate: return True timeout.sleep() return False
def verify_services_accounting_status(device, expected_export_format=None, route_record_threshold=None, max_time=60, check_interval=10): """ Verify 'show services accounting status' against criteria Args: device (obj): Device object expected_export_format (str): Session address route_record_threshold (str): Expected session state max_time (int, optional): Maximum timeout time. Defaults to 60. check_interval (int, optional): Check interval. Defaults to 10. Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = None try: out = device.parse('show services accounting status') except SchemaEmptyParserError: timeout.sleep() continue #"status-information": [ # { # "interface-name": "ms-9/0/0", # "status-export-format": "9", # "status-route-record-count": "902244", # "status-ifl-snmp-map-count": "296", # "status-as-count": "0", # "status-monitor-config-set": "Yes", # "status-monitor-route-record-set": "No", # "status-monitor-ifl-snmp-set": "Yes", # } if expected_export_format and \ out.q.get_values('status-export-format',0) != str(expected_export_format): timeout.sleep() continue if route_record_threshold and \ int(out.q.get_values('status-route-record-count',0)) < int(route_record_threshold): timeout.sleep() continue return True return False
def verify_bgp_last_error(device, interface, expected_error, max_time=60, check_interval=10): """ Verify bgp last error Args: device('obj'): device to use interface('str'): Peer interface expected_error('str') : Expected last error max_time ('int', optional): Maximum time to keep checking. Default to 60 check_interval ('int', optional): How often to check. Default to 10 Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) # show commands: "show bgp neighbor" # {'bgp-information': {'bgp-peer': [{ # 'flap-count': '0', # 'peer-address': '20.0.0.3+63208', # 'last-error': 'None', # . # . # . while timeout.iterate(): try: out = device.parse('show bgp neighbor') except SchemaEmptyParserError: timeout.sleep() continue peers_list = out.q.get_values("bgp-peer") for peer in peers_list: peer_interface = peer.get('peer-address') error_msg = peer.get("last-error") # 20.0.0.3+63208 if '+' in peer_interface: peer_interface = peer_interface.split('+')[0] if peer_interface == interface and error_msg == expected_error: return True timeout.sleep() return False
def verify_ospf_overview(device, router_id=None, max_time=90, check_interval=10, expected_configured_overload=None): """ Verifies ospf overview values Args: device ('obj'): device to use router_id ('str'): Router ID max_time ('int'): Maximum time to keep checking check_interval ('int'): How often to check expected_configured_overload ('str'/'int'): Configured overload time or * for any Returns: True/False Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = None try: out = device.parse('show ospf overview') except SchemaEmptyParserError: timeout.sleep() continue if router_id: router_id_ = out.q.contains_key_value('ospf-router-id', router_id, value_regex=True) if router_id_: return True if expected_configured_overload: configured_overload = out.q.get_values( 'ospf-configured-overload-remaining-time', None) if configured_overload and expected_configured_overload == "*": return True if configured_overload: try: if int(configured_overload) == int( expected_configured_overload): return True except ValueError as e: log.info("Non-integer value given") raise timeout.sleep() return False
def verify_rsvp_session_state(device, expected_state, session_name=None, session_type="Transit", max_time=60, check_interval=10): """ Verify RSVP session state Args: device (obj): device object expected_state (str): Expected state session_name (str, optional): Session name. Defaults to None. session_type (str): Session type. Defaults to "Transit" max_time (int, optional): Maximum timeout time. Defaults to 60. check_interval (int, optional): Check interval. Defaults to 10. """ #'rsvp-session-information': { # 'rsvp-session-data': [{ # 'session-type': 'Transit', # 'count': '30', # 'rsvp-session': [{ # 'destination-address': '10.49.194.125', # 'source-address': '10.49.194.127', # 'lsp-state': 'Up', # 'route-count': '0', # 'rsb-count': '1', # 'resv-style': 'FF', # 'label-in': '46', # 'label-out': '44', # 'name': 'test_lsp_01' # }, timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse('show rsvp session') except SchemaEmptyParserError: timeout.sleep() continue for session in out.q.get_values('rsvp-session-data'): if session.get('session-type') == session_type: session_data = Dq(session).get_values('rsvp-session') for data in session_data: if session_name and session_name != data.get('name'): continue if data.get('lsp-state').lower() != expected_state.lower(): continue return True timeout.sleep() return False
def verify_ospf_neighbors_found(device, extensive=False, max_time=90, check_interval=10, expected_interface=None, instance=None): """ Verifies ospf neighbors values exists Args: device ('obj'): device to use extensive ('str'): If to check with extensive command max_time ('int'): Maximum time to keep checking check_interval ('int'): How often to check expected_interface ('str'): Interface to check for instance ('str'): Instance to check for Returns: True/False Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = None try: if instance: out = device.parse( 'show ospf neighbor instance {instance}'.format( instance=instance)) elif extensive: out = device.parse('show ospf neighbor extensive') else: out = device.parse('show ospf neighbor') except SchemaEmptyParserError: timeout.sleep() continue ospf_neighbors = out.q.get_values('ospf-neighbor') if expected_interface: if len([ neighbor for neighbor in ospf_neighbors if neighbor.get('interface-name') == expected_interface ]) > 0: return True else: timeout.sleep() continue if len(ospf_neighbors) > 0: return True timeout.sleep() return False
def verify_interfaces_terse_state(device, interface, expected_admin_state=None, expected_link_state=None, expected_oper_status=None, max_time=30, check_interval=10, expected_result=True): """ Verify interfaces terse Args: device (`obj`): Device object interface (`str`): Interface name expected_admin_state (`str`): Expected admin state for interface ex.) expected_admin_state = 'up' expected_link_state (`str`): Expected link state for interface ex.) expected_link_state = 'down' expected_oper_status (`str`): Expected oper state for interface ex.) expected_oper_status = 'up' Returns: result (`bool`): Verified result Raises: N/A """ timeout = Timeout(max_time, check_interval) interface_terse_out = None result = True while timeout.iterate(): try: if interface: interface_terse_out = device.parse('show interfaces {interface} terse'.format( interface=interface)) else: interface_terse_out = device.parse('show interfaces terse') result = True except SchemaEmptyParserError: log.info('Failed to parse. Device output might contain nothing.') if not expected_result: return False result = False timeout.sleep() continue for intf, intf_dict in interface_terse_out.items(): admin_state = intf_dict.get('admin_state', None) link_state = intf_dict.get('link_state', None) oper_status = intf_dict.get('oper_status', None) enabled = intf_dict.get('enabled', None) if expected_admin_state and admin_state != expected_admin_state: result = False if expected_link_state and link_state != expected_link_state: result = False if expected_oper_status and oper_status != expected_oper_status: result = False if result == expected_result: return expected_result timeout.sleep() return False
def verify_bgp_soo_on_route( device, address_family, route, soo_rt, vrf=None, rd=None, max_time=60, check_interval=15, ): """ Verifies soo exists on route Args: Required: device('obj'): device to verify on address_family('str'): address_family to verify under route('str'): route to verify One or the other: vrf('str'): verify using vrf rd('str'): verify using rd Optional: timeout('obj'): timeout object to override default Returns: True False Raise: None """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): if vrf: result = get_bgp_route_ext_community(device, address_family, route, vrf=vrf) if not result: return False elif rd: result = get_bgp_route_ext_community(device, address_family, route, rd=rd) if not result: return False if result and "SoO:{}".format(soo_rt) in result: return True timeout.sleep() return False
def verify_ntp_association_with_server( device, ip_address_server, peer_mode, max_stratum, max_time=15, check_interval=5 ): """Verify association with server Args: server (`obj`): Server Device object ip_address_server (`str`): IP address to server peer_mode (`str`): peer mode type max_stratum (`int`): maximum stratum value max_time (int): Maximum wait time for the trigger, in seconds. Default: 15 check_interval (int): Wait time between iterations when looping is needed, in seconds. Default: 5 Returns: True False """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): result = False try: out = device.parse("show ntp associations detail") except SchemaEmptyParserError as e: log.error("Parser not found for command " "'show ntp associations detail'") return False for vrf in out["vrf"].keys(): try: allowed_ip_dict = out["vrf"][vrf]["associations"]["address"][ ip_address_server ]["local_mode"]["client"]["isconfigured"]["True"] stratum_found = allowed_ip_dict["stratum"] refid = allowed_ip_dict["refid"] peer_mode_found = allowed_ip_dict["peer"][refid]["local_mode"][ peer_mode ]["local_mode"] except KeyError as e: log.error( "Could not find following key{}, exception {}".format( peer_mode, str(e) ) ) result = False if stratum_found < max_stratum and peer_mode_found == peer_mode: result = True else: result = False if result: return result timeout.sleep() return result
def verify_routing_neighbors_advertised_routes_exists( device, address_family, neighbor, vrf, initial_route, rd=None, max_time=60, check_interval=10, output=None, ): """ Verify if neighbors advertised routes exists Args: device ('obj'): Device object address_family ('str'): address family rd ('str'): rd export value neighbor ('str'): neighbor address to find routes vrf ('str'): vrf name initial_route ('str'): intial route to search max_time ('int') : max time for Timeout check_interval ('int'): interval for Timeout output ('list'): list of neighbors advertised routes (get_bgp_neighbors_advertised_routes) Returns: True False """ result = False timeout = Timeout(max_time=max_time, interval=check_interval) while timeout.iterate(): if not output: advertised_routes = list( get_bgp_neighbors_advertised_routes( device=device, address_family=address_family, neighbor_address=neighbor, rd=rd, vrf=vrf, )) if not advertised_routes: return False else: advertised_routes = output if advertised_routes: for route in advertised_routes: if route == initial_route: result = True return result else: timeout.sleep() return result
def verify_ospf_neighbor_state(device, expected_state, interface, neighbor_address=None, extensive=False, max_time=60, check_interval=10): """ Verifies state of ospf neighbor Args: device ('obj'): device to use expected_state ('str'): OSPF adjacency state that is expected interface ('str'): Name of interface neighbor_address ('str'): Neighbor address extensive ('bool'): If ospf command is extensive max_time ('int'): Maximum time to keep checking check_interval ('int'): How often to check Returns: True/False Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: if extensive: output = device.parse('show ospf neighbor extensive') else: output = device.parse('show ospf neighbor') except SchemaEmptyParserError: timeout.sleep() continue #{'ospf-neighbor-information': {'ospf-neighbor': [{}]}} neighbors = output.q.get_values('ospf-neighbor') for neighbor in neighbors: #'interface-name': 'ge-0/0/0.0' #'ospf-neighbor-state': 'Full' if not neighbor_address: if neighbor.get('interface-name',[]) == interface and \ neighbor.get('ospf-neighbor-state',[]).lower() == expected_state.lower(): return True else: if neighbor.get('neighbor-address',[]) == neighbor_address and \ neighbor.get('ospf-neighbor-state',[]).lower() == expected_state.lower(): return True timeout.sleep() return False
def verify_interface_errors(device, interface, expected_value=None, input=False, output=False, max_time=30, check_interval=10): """ Verify interface input and output errors Args: device (`obj`): Device object interface (`str`): Pass interface in show command expected_value (`int`, Optional): Expected errors values. Defaults to None input (`bool`, Optional): True if input errors to verify. Default to False. output (`bool`, Optional): True if output errors to verify. Default to False. max_time (`int`, Optional): Max time, default: 60 seconds check_interval (`int`, Optional): Check interval, default: 10 seconds Returns: result (`bool`): Verified result Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: cmd = 'show interface {interface}'.format(interface=interface) out = device.parse(cmd) except SchemaEmptyParserError: timeout.sleep() continue interface, data = next(iter(out.items())) data = Dq(data) if input and output: input_errors = data.get_values("in_errors", 0) output_errors = data.get_values("out_errors", 0) if input_errors == output_errors == expected_value: return True elif input: input_errors = data.get_values("in_errors", 0) if input_errors == expected_value: return True elif output: output_errors = data.get_values("out_errors", 0) if output_errors == expected_value: return True timeout.sleep() continue return False
def verify_ospf_advertising_router_metric_in_database(device, lsa_id, ospf_link_id, expected_metric, max_time=60, check_interval=10): """ Verifies ospf advertising router metric in database Args: device ('obj'): Device to use lsa_id: lsa_id to check ospf_link_id ('str'): Ospf link id to check expected_metric ('str'): Metric of desired ospf link max_time ('int'): Maximum time to keep checking check_interval ('int'): How often to check Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: cmd = 'show ospf database advertising-router {ipaddress} extensive'.format( ipaddress=lsa_id) out = device.parse(cmd) except SchemaEmptyParserError: timeout.sleep() continue # Example output: # { # "ospf-database-information": { # "ospf-database": [ # { # "ospf-router-lsa": { # "ospf-link": [ # { # "link-id": "106.187.14.240", # "metric": "1" # }]}}]}} for ospf_database in out.q.get_values('ospf-database'): ospf_links = Dq(ospf_database).get_values('ospf-link') for ospf_link in ospf_links: if ospf_link.get('link-id', None) == ospf_link_id \ and ospf_link.get('metric', None) == str(expected_metric): return True timeout.sleep() return False
def verify_bgp_all_peer_state(device, expected_state, max_time=60, check_interval=10): """ Verify bgp all peer state Args: device('obj'): device to use expected_state('str') : Expected peer state Default to `Establ` max_time ('int', optional): Maximum time to keep checking. Default to 60 check_interval ('int', optional): How often to check. Default to 10 Returns: Boolean Raises: N/A """ # show commands: "show bgp summary" # { # "bgp-information": { # "bgp-peer": [ # { # (snip) # "flap-count": "0", # "input-messages": "4", # "output-messages": "4", # "peer-address": "10.0.0.2", # "peer-as": "65002", # "peer-state": "Establ", # "route-queue-count": "0" # }, timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse('show bgp summary') except SchemaEmptyParserError: timeout.sleep() continue special_case_state = re.compile(r'([\d\/]+\s+[\d\/]+)') peer_state = [val if not special_case_state.match(val) else 'Establ' for val in out.q.get_values('peer-state')] if set([expected_state]) == set(peer_state): return True timeout.sleep() return False
def verify_bfd_session(device, session_address, expected_session_state=None, expected_session_multiplier=None, max_time=60, check_interval=10, expected_interface=None): """ Verifiy the session state Args: device (obj): Device object session_address (str): Session address expected_session_state (str): Expected session state expected_session_multiplier (str): Expected session multiplier max_time (int, optional): Maximum timeout time. Defaults to 60. check_interval (int, optional): Check interval. Defaults to 10. expected_interface (str, optional): Expected interface to check Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = None try: out = device.parse('show bfd session') except SchemaEmptyParserError: timeout.sleep() continue sessions_ = out.q.get_values('bfd-session') for session in sessions_: if session.get('session-neighbor') == session_address: if expected_session_multiplier and \ session.get('session-adaptive-multiplier') != str(expected_session_multiplier): continue if expected_session_state and \ session.get('session-state').lower() != expected_session_state.lower(): continue if expected_interface and session.get( 'session-interface') != expected_interface: continue return True timeout.sleep() return False
def verify_lacp_interface_receive_state( device, interface, expected_state, expected_interface=None, max_time=60, check_interval=10, ): """Verify the state of an lackp interface Args: device (obj): Device object interface (str): Interface name. Will be used if expected_interface isn't set expected_state (str): Expected state to check against. Defaults to None. expected_interface (str, optional): Expected interface to check against. Defaults to None. max_time (int, optional): Maximum timeout time. Defaults to 60. check_interval (int, optional): Check interval. Defaults to 10. """ interface_to_use = expected_interface if expected_interface else interface timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse( 'show lacp interfaces {interface}'.format(interface=interface)) except SchemaEmptyParserError: timeout.sleep() continue # { # "lacp-interface-information-list": { # "lacp-interface-information": { # "lag-lacp-protocol": [ # { # "lacp-mux-state": "Collecting " "distributing", # "lacp-receive-state": "Current", # "lacp-transmit-state": "Fast " "periodic", # "name": "ge-0/0/0", # }, interfaces = out.q.get_values('lag-lacp-protocol') states = { intr['name']: intr['lacp-receive-state'] for intr in interfaces } if expected_state == states.get(interface_to_use, None): return True timeout.sleep() return False
def verify_bgp_holdtime(device, expected_holdtime, interface, max_time=60, check_interval=10): """ Verify bgp holdtimer with peer {interface} Args: device('obj'): device to use interface('str'): Peer interface expected_holdtime('str'): Expected holdtime max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds check_interval ('int', optional): How often to check. Default to 10 seconds Returns: Boolean Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse('show bgp neighbor') except SchemaEmptyParserError: timeout.sleep() continue filter_output = out.q.contains('holdtime|peer-address', regex=True).reconstruct() # {'bgp-information': { # 'bgp-peer': [ # {'bgp-option-information': { # 'holdtime': '30'}, # 'peer-address': '20.0.0.3+179'}, # {'bgp-option-information': { # 'holdtime': '10'}, # 'peer-address': '2001:20::3+179'}]}} peer_list = filter_output.get('bgp-information').get('bgp-peer') for peer in peer_list: # 20.0.0.3+63208 peer_address = peer.get('peer-address').split('+')[0] if peer_address == interface and\ Dq(peer).get_values('holdtime')[0] == str(expected_holdtime): return True timeout.sleep() return False
def verify_bgp_peer_import_value(device: object, peer_address: str, expected_import_value: str, max_time: int = 60, check_interval: int = 10) -> bool: """Verifies BGP peer import value Args: device (object): Device object peer_address (str): Peer address expected_import_value (str): Expected import value to check against max_time (int, optional): Maximum timeout time. Defaults to 60. check_interval (int, optional): Check interval. Defaults to 10. Returns: bool: True/False """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse('show bgp neighbor') except SchemaEmptyParserError: timeout.sleep() continue # Example dict # "bgp-information": { # "bgp-peer": [ # { # "bgp-option-information": { # "import-policy": str, for peer in out.q.get_values('bgp-peer'): peer_interface_ = peer.get('peer-address') # 20.0.0.3+63208 if '+' in peer_interface_: peer_interface_ = peer_interface_.split('+')[0] # 20.0.0.2/24 if '/' in peer_address: peer_address = peer_address.split('/')[0] if peer_interface_ != peer_address: continue if Dq(peer).get_values('import-policy', 0) == expected_import_value: return True timeout.sleep() return False
def verify_ospf_metric(device, interface, metric, max_time=60, check_interval=10): """Verify the OSPF metric Args: device (obj): Device object interface (str): Interface name metric (str): OSPF metric Returns: True/False Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = None try: out = device.parse('show ospf interface extensive') except SchemaEmptyParserError: timeout.sleep() continue # Example dictionary # "ospf-interface": [ # { # "interface-name": "ge-0/0/0.0", # "ospf-interface-topology": { # "ospf-topology-metric": "5", # }, # }, ospf_interface_list = Dq(out).get_values('ospf-interface') for ospf_interface_dict in ospf_interface_list: #{'interface-name': 'ge-0/0/1.0'} interface_ = Dq(ospf_interface_dict).get_values( 'interface-name', 0) #{'ospf-topology-metric': '5'} metric_ = Dq(ospf_interface_dict).get_values( 'ospf-topology-metric', 0) if interface_.lower() == interface.lower() and str(metric_) == str( metric): return True timeout.sleep() return False
def verify_smallest_stratum_ntp_system_peer(device, max_time=90, check_interval=15): """ Verify NTP server with the smallest stratum is elected as system peer Args: device (`obj`): Device object max_time (int): Maximum wait time for the trigger, in seconds. Default: 90 check_interval (int): Wait time between iterations when looping is needed, in seconds. Default: 15 Returns: result (`bool`): Verified result sys_peer (`str`): System peer ip other_peers (`list`): Other peers ip """ result = "" sys_peer = "" other_peers = [] timeout = Timeout(max_time, check_interval) while timeout.iterate(): out = device.parse("show ntp associations") stratum = ( out.get("clock_state", {}).get("system_status", {}).get("clock_stratum") ) sys_peer = ( out.get("clock_state", {}) .get("system_status", {}) .get("associations_address") ) for peer, data in out["peer"].items(): if peer != sys_peer: other_peers.append(peer) peer_stratum = data["local_mode"]["client"]["stratum"] if stratum > peer_stratum: log.info( "Found peer {peer} has a smaller stratum {stratum}".format( peer=peer, stratum=peer_stratum ) ) result = False break else: result = True if not result: timeout.sleep() else: break return result, sys_peer, other_peers
def verify_module_status(device, timeout=180, interval=30, ignore_modules=None): ''' Check status of slot using 'show platform' Args: device ('obj'): Device object timeout ('int'): Max timeout to re-check slot status interval ('int'): Max interval to re-check slot status ignore_modules ('list'): Modules to ignore status check ''' timeout = Timeout(max_time=timeout, interval=interval) while timeout.iterate(): # Reset failed_slots = [] try: output = device.parse("show platform") except SchemaEmptyParserError: timeout.sleep() continue # Check state for all slots failed_slots = Dq(output).contains('state').\ not_contains_key_value('state', 'empty').\ not_contains_key_value('state', '.*ok.*|standby|ha-standby|Ready', value_regex=True).\ get_values('slot') # To ignore specified modules if ignore_modules: for module in ignore_modules: if module in failed_slots: failed_slots.remove(module) log.info("Ignoring the following modules '{}'".\ format(ignore_modules)) if not failed_slots: log.info("All modules on '{}' are in stable state".\ format(device.name)) break else: log.warning("The following modules are not in stable state {}".\ format(failed_slots)) log.warning( "Sleeping {} seconds before rechecking".format(interval)) timeout.sleep() continue else: raise Exception("Modules on '{}' are not in stable state".\ format(device.name))
def verify_specific_route(device, address, learn_protocol, max_time=60, check_interval=10): """Verifies address list agianst 'show route protocol {protocol}' Args: device ('obj'): device to use address('str'): address to search for learn_protocol('str'): Learned protocol max_time ('int'): Maximum time to keep checking check_interval ('int'): How often to check Returns: True/False Raises: N/A """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: output = device.parse( "show route {address}".format(address=address)) except SchemaEmptyParserError: timeout.sleep() continue # [{'rt-entry': {'active-tag': '*', 'protocol-name': 'OSPF', # 'preference': '10', 'metric': '1', 'age': {'#text': '00:00:01'}, # 'nh': [{'to': '106.187.14.158', 'via': 'ge-0/0/0.0'}]}, # 'rt-destination': '59.128.2.250/32'}] rt_destination_ = Dq(output).get_values("rt-destination", 0) # [{'rt-entry': {'active-tag': '*', 'protocol-name': 'OSPF', # 'preference': '10', 'metric': '1', 'age': {'#text': '00:00:01'} protocol_name = Dq(output).get_values("protocol-name", 0) if protocol_name: if rt_destination_.startswith(str(address)) and \ protocol_name.lower() == learn_protocol.lower(): return True else: timeout.sleep() continue timeout.sleep() return False
def verify_bfd_session_destination_detail( device, address, expected_session_state=None, expected_received_parameters_state=None, max_time=60, check_interval=10, ): """ Verifies bfd session destination detail Args: device ('obj'): device to use address ('str'): IP address for command expected_session_state ('str'): Session state to verify expected_received_parameters_state ('str'): Received parameter state to verify max_time ('int'): Max time to retry. Default to 60 check_interval ('int'): Number of check in interval. Default to 10 Returns: True False Raises: None """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse( "show bfd session destination {address} detail".format( address=address ) ) except SchemaEmptyParserError: timeout.sleep() continue if expected_session_state: state = out.q.contains('session').get_values('state', 0) if state != expected_session_state: timeout.sleep() continue if expected_received_parameters_state: received_parameters_state = out.q.contains( 'received_parameters').get_values('state', 0) if received_parameters_state != expected_received_parameters_state: timeout.sleep() continue timeout.sleep() return False
def verify_device_tracking_counters_vlan_dropped(device, vlanid, feature, protocol, message, num_dropped, max_time=20, check_interval=10): """ Verify missing ipv6 source guard configurations Args: device('obj'): device object vlanid('str'): vlan id feature('str'): feature protocol('str'): protocol message('str'): message type num_dropped('int'): number of dropped packets max_time('int', optional): max check time. Defaults to 20 check_interval('int', optional): check intervals. Defaults to 10 Returns: True False Raises: None """ timeout = Timeout(max_time, check_interval) while timeout.iterate(): output = device.parse( 'show device-tracking counters vlan {vlanid}'.format( vlanid=vlanid)) dropped = output.get('vlanid', {}).get(int(vlanid), {}) \ .get('dropped', {}).get(feature, {}) if dropped: if dropped['protocol'] == protocol and dropped['message'] == message and \ dropped['dropped'] == num_dropped: return True else: log.info( 'Expected protocol: {protocol}, message: {message}, dropped: {dropped}. ' 'Instead got protocol: {target_protocol}, message: {target_message}, ' 'dropped: {target_dropped}.'.format( protocol=protocol, message=message, dropped=num_dropped, target_protocol=dropped['protocol'], target_message=dropped['message'], target_dropped=dropped['dropped'])) return False timeout.sleep() log.info('Dropped message type cannot be found') return False
def verify_mpls_forwarding_table_has_prefix_in_subnet_range( device, subnet, max_time=120, check_interval=30): """ Verifies local label for entries with a prefix inside subnet Args: device ('obj'): Device to use subnet ('str'): Subnet to verify inside max_time ('int'): Max time to check check_interval ('int'): How often to check returns: True/False raises: N/A """ log.info( 'Checking atleast one entry has a prefix in subnet {subnet} range'. format(subnet=subnet)) try: subnet = IPNetwork(subnet) except Exception: log.info('Bad subnet provided') return False timeout = Timeout(max_time, check_interval) while timeout.iterate(): try: out = device.parse('show mpls forwarding-table') except SchemaEmptyParserError: log.info('Parser output is empty') timeout.sleep() continue for vrf in out.get('vrf'): for local_label in out['vrf'][vrf].get('local_label'): for out_label in out['vrf'][vrf]['local_label'][local_label]\ .get('outgoing_label_or_vc'): for prefix in out['vrf'][vrf]['local_label'][local_label]\ ['outgoing_label_or_vc'][out_label]\ .get('prefix_or_tunnel_id'): try: pfx = IPNetwork(prefix) except Exception: continue if pfx in subnet: return True timeout.sleep() return False
def execution(self, device, **kwargs): # Init status = OK # create timeout object timeout = Timeout(max_time=int(self.args.cpucheck_timeout), interval=int(self.args.cpucheck_interval)) # loop status loop_stat_ok = True if not hasattr(self, 'PARSER_MODULE'): return WARNING('Does not have CPU related parsers to check') while timeout.iterate(): # Execute command to get five minutes usage percentage try: cpu_dict = self.PARSER_MODULE(device).parse(sort_time='5min', key_word='CPU') except Exception as e: return ERRORED( 'No output from show processes cpu\n{}'.format(e)) # Check 5 minutes percentage smaller than cpucheck_fivemin_pcnt if int(cpu_dict['five_min_cpu']) >= int( self.args.cpucheck_fivemin_pcnt): message = "****** Device {d} *****\n".format(d=device.name) message += "Excessive CPU utilization detected for 5 min interval\n" message += "Allowed: {e}%\n".format( e=self.args.cpucheck_fivemin_pcnt) message += "Measured: FiveMin: {r}%".format( r=cpu_dict['five_min_cpu']) loop_stat_ok = False timeout.sleep() else: message = "***** CPU usage is Expected ***** \n" message += "Allowed threashold: {e} \n"\ .format(e=self.args.cpucheck_fivemin_pcnt) message += "Measured from device: {r}"\ .format(r=cpu_dict['five_min_cpu']) loop_stat_ok = True status += OK(message) logger.info(banner(message)) break if not loop_stat_ok: status += CRITICAL(message) logger.error(banner(message)) # Final status return status