def _reconnect(self, steps, timeout):
'''Reconnect to the device if needed'''
if self.process in self.reconnect:
ha = self.abstract.sdk.libs.abstracted_libs.ha.HA(
device=self.device)
with steps.start(
'The device is reloading when restarting this process',
continue_=True) as step:
# Waiting 4-7 mins for switchover to complete
log.info(
"Waiting for a maximum of 7 minutes for device switchover to completess"
)
check_time = Timeout(max_time=700, interval=20)
while check_time.iterate():
check_time.sleep()
continue
self.device.destroy()
time.sleep(30)
temp = TempResult(container=step)
while timeout.iterate():
try:
self.device.connect(prompt_recovery=True)
except Exception as e:
temp.failed('Could not reconnect to the device',
from_exception=e)
timeout.sleep()
continue
temp.passed('Reconnected to the device')
break
temp.result()
# check show module
with steps.start('Check module status after reconnection',
continue_=True) as step:
temp = TempResult(container=step)
while timeout.iterate():
try:
ha.check_module()
except AttributeError as e:
temp.failed(
'Could not find mandatory information from show module',
from_exception=e)
continue
except AssertionError as e:
temp.failed('Modules are not ready', from_exception=e)
timeout.sleep()
continue
temp.passed('Modules are ready')
break
temp.result()
def verify_tunnel_stats(
device,
tunnel_id,
direction,
tx_packets,
pkt_rate,
max_time=3,
check_interval=5
):
""" Verify acl usage
Args:
device (`obj`): Device object
tunnel_id (`str`): Tunnel id
direction (`str`): Traffic direction
tx_packets (`int`): Transmit packets
pkt_rate (`str`): Frames per seconds
max_time ('int',optional): Maximum wait time for the trigger,
in second. Default: 3
check_interval (`int`, optional): Wait time between iterations when looping is needed,
in second. Default: 5
Returns:
True
False
"""
out = None
try:
out = device.parse("show interfaces {intf} stats".format(intf=tunnel_id))
except SchemaEmptyParserError:
pass
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
if out:
get_packet_count_in = out[tunnel_id]['switching_path']['distributed_cache']['pkts_in']
get_packet_count_out = out[tunnel_id]['switching_path']['distributed_cache']['pkts_out']
expected_max_pkts = tx_packets + int(pkt_rate)
expected_min_pkts = tx_packets - int(pkt_rate)
if direction == 'egress':
if not get_packet_count_out > 0:
return False
if (get_packet_count_out >= expected_min_pkts) and (get_packet_count_out <= expected_max_pkts):
return True
if direction == 'ingress':
if not get_packet_count_in > 0:
return False
if (get_packet_count_in >= expected_min_pkts) and (get_packet_count_in <= expected_max_pkts):
return True
timeout.sleep()
return False
def verify_bgp_peer_as(device,
peer_address,
expected_peer_as,
max_time=60,
check_interval=10):
"""
Verify bgp peer AS number
Args:
device('obj'): device to use
peer_address('str'): Peer interface
expected_peer_as ('int'): Expected peer AS number
max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds.
check_interval ('int', optional): How often to check. Default to 10 seconds.
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp neighbor"
# {
# "bgp-information": {
# "bgp-peer": [
# {
# "local-address": "10.189.5.252",
# "local-as": "65171",
# "peer-address": "10.49.216.179",
# "peer-as": "65171", <-----------------------
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
for peer in peers_list:
addr = peer.get('peer-address')
peer_as = Dq(peer).get_values('peer-as', 0)
# 20.0.0.3+63208
addr = addr.split('+')[0]
if addr == peer_address.split('/')[0] and int(
peer_as) == expected_peer_as:
return True
timeout.sleep()
return False
def verify_no_ospf_interface_in_database(device,
expected_interface,
max_time=60,
check_interval=10):
""" Verifies ospf interface exists with criteria
Args:
device ('obj'): device to use
expected_interface ('str'): Interface to use
max_time ('int', optional): Maximum time to keep checking. Defaults to 60 seconds.
check_interval (`int`): Check interval, default: 10
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
exist = False
try:
out = device.parse('show ospf database extensive')
except SchemaEmptyParserError:
timeout.sleep()
continue
#'ospf-database':[
# {'lsa-type':
# 'Router',
# 'lsa-id': '1.1.1.1'
# }
#]
for ospf_database in Dq(out).get_values('ospf-database'):
#'ospf-external-lsa':
# {'address-mask': '255.255.255.255',
# 'ospf-external-lsa-topology': {
# 'ospf-topology-name':
# 'default'}}
ospf_external_lsa = ospf_database.get('ospf-external-lsa', {})
advertising_router = ospf_database.get('advertising-router', None)
if expected_interface == advertising_router:
exist = True
if exist:
timeout.sleep()
continue
else:
return False
return True
def verify_ping(
device,
address,
expected_max_success_rate=100,
expected_min_success_rate=0,
count=None,
source=None,
max_time=60,
check_interval=10,
):
"""Verify ping
Args:
device ('obj'): Device object
address ('str'): Address value
expected_max_success_rate (int): Expected maximum success rate
expected_min_success_rate (int): Expected minimum success rate
count ('int'): Count value for ping command
source ('str'): Source IP address, default: None
max_time (`int`): Max time, default: 30
check_interval (`int`): Check interval, default: 10
"""
p = re.compile(r"Success +rate +is +(?P<rate>\d+) +percent.*")
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
if address and count and source:
cmd = 'ping {address} source {source} repeat {count}'.format(
address=address, source=source, count=count)
elif address and count:
cmd = 'ping {address} repeat {count}'.format(address=address,
count=count)
elif address and source:
cmd = 'ping {address} source {source}'.format(address=address,
source=source)
elif address:
cmd = 'ping {address}'.format(address=address)
else:
log.info('Need to pass address as argument')
return False
try:
out = device.execute(cmd)
except SubCommandFailure as e:
timeout.sleep()
continue
rate = int(p.search(out).groupdict().get('rate', 0))
if expected_max_success_rate >= rate >= expected_min_success_rate:
return True
timeout.sleep()
return False
def verify_services_accounting_status(device,
expected_export_format=None,
route_record_threshold=None,
max_time=60,
check_interval=10):
""" Verify 'show services accounting status' against criteria
Args:
device (obj): Device object
expected_export_format (str): Session address
route_record_threshold (str): Expected session state
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show services accounting status')
except SchemaEmptyParserError:
timeout.sleep()
continue
#"status-information": [
# {
# "interface-name": "ms-9/0/0",
# "status-export-format": "9",
# "status-route-record-count": "902244",
# "status-ifl-snmp-map-count": "296",
# "status-as-count": "0",
# "status-monitor-config-set": "Yes",
# "status-monitor-route-record-set": "No",
# "status-monitor-ifl-snmp-set": "Yes",
# }
if expected_export_format and \
out.q.get_values('status-export-format',0) != str(expected_export_format):
timeout.sleep()
continue
if route_record_threshold and \
int(out.q.get_values('status-route-record-count',0)) < int(route_record_threshold):
timeout.sleep()
continue
return True
return False
def verify_bgp_last_error(device,
interface,
expected_error,
max_time=60,
check_interval=10):
"""
Verify bgp last error
Args:
device('obj'): device to use
interface('str'): Peer interface
expected_error('str') : Expected last error
max_time ('int', optional): Maximum time to keep checking. Default to 60
check_interval ('int', optional): How often to check. Default to 10
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# 'flap-count': '0',
# 'peer-address': '20.0.0.3+63208',
# 'last-error': 'None',
# .
# .
# .
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
for peer in peers_list:
peer_interface = peer.get('peer-address')
error_msg = peer.get("last-error")
# 20.0.0.3+63208
if '+' in peer_interface:
peer_interface = peer_interface.split('+')[0]
if peer_interface == interface and error_msg == expected_error:
return True
timeout.sleep()
return False
def verify_ospf_overview(device,
router_id=None,
max_time=90,
check_interval=10,
expected_configured_overload=None):
""" Verifies ospf overview values
Args:
device ('obj'): device to use
router_id ('str'): Router ID
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
expected_configured_overload ('str'/'int'): Configured overload time or * for any
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show ospf overview')
except SchemaEmptyParserError:
timeout.sleep()
continue
if router_id:
router_id_ = out.q.contains_key_value('ospf-router-id',
router_id,
value_regex=True)
if router_id_:
return True
if expected_configured_overload:
configured_overload = out.q.get_values(
'ospf-configured-overload-remaining-time', None)
if configured_overload and expected_configured_overload == "*":
return True
if configured_overload:
try:
if int(configured_overload) == int(
expected_configured_overload):
return True
except ValueError as e:
log.info("Non-integer value given")
raise
timeout.sleep()
return False
def verify_rsvp_session_state(device,
expected_state,
session_name=None,
session_type="Transit",
max_time=60,
check_interval=10):
""" Verify RSVP session state
Args:
device (obj): device object
expected_state (str): Expected state
session_name (str, optional): Session name. Defaults to None.
session_type (str): Session type. Defaults to "Transit"
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
"""
#'rsvp-session-information': {
# 'rsvp-session-data': [{
# 'session-type': 'Transit',
# 'count': '30',
# 'rsvp-session': [{
# 'destination-address': '10.49.194.125',
# 'source-address': '10.49.194.127',
# 'lsp-state': 'Up',
# 'route-count': '0',
# 'rsb-count': '1',
# 'resv-style': 'FF',
# 'label-in': '46',
# 'label-out': '44',
# 'name': 'test_lsp_01'
# },
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show rsvp session')
except SchemaEmptyParserError:
timeout.sleep()
continue
for session in out.q.get_values('rsvp-session-data'):
if session.get('session-type') == session_type:
session_data = Dq(session).get_values('rsvp-session')
for data in session_data:
if session_name and session_name != data.get('name'):
continue
if data.get('lsp-state').lower() != expected_state.lower():
continue
return True
timeout.sleep()
return False
def verify_ospf_neighbors_found(device,
extensive=False,
max_time=90,
check_interval=10,
expected_interface=None,
instance=None):
""" Verifies ospf neighbors values exists
Args:
device ('obj'): device to use
extensive ('str'): If to check with extensive command
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
expected_interface ('str'): Interface to check for
instance ('str'): Instance to check for
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
if instance:
out = device.parse(
'show ospf neighbor instance {instance}'.format(
instance=instance))
elif extensive:
out = device.parse('show ospf neighbor extensive')
else:
out = device.parse('show ospf neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
ospf_neighbors = out.q.get_values('ospf-neighbor')
if expected_interface:
if len([
neighbor for neighbor in ospf_neighbors
if neighbor.get('interface-name') == expected_interface
]) > 0:
return True
else:
timeout.sleep()
continue
if len(ospf_neighbors) > 0:
return True
timeout.sleep()
return False
def verify_interfaces_terse_state(device, interface, expected_admin_state=None, expected_link_state=None, expected_oper_status=None, max_time=30, check_interval=10, expected_result=True):
""" Verify interfaces terse
Args:
device (`obj`): Device object
interface (`str`): Interface name
expected_admin_state (`str`): Expected admin state for interface
ex.) expected_admin_state = 'up'
expected_link_state (`str`): Expected link state for interface
ex.) expected_link_state = 'down'
expected_oper_status (`str`): Expected oper state for interface
ex.) expected_oper_status = 'up'
Returns:
result (`bool`): Verified result
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
interface_terse_out = None
result = True
while timeout.iterate():
try:
if interface:
interface_terse_out = device.parse('show interfaces {interface} terse'.format(
interface=interface))
else:
interface_terse_out = device.parse('show interfaces terse')
result = True
except SchemaEmptyParserError:
log.info('Failed to parse. Device output might contain nothing.')
if not expected_result:
return False
result = False
timeout.sleep()
continue
for intf, intf_dict in interface_terse_out.items():
admin_state = intf_dict.get('admin_state', None)
link_state = intf_dict.get('link_state', None)
oper_status = intf_dict.get('oper_status', None)
enabled = intf_dict.get('enabled', None)
if expected_admin_state and admin_state != expected_admin_state:
result = False
if expected_link_state and link_state != expected_link_state:
result = False
if expected_oper_status and oper_status != expected_oper_status:
result = False
if result == expected_result:
return expected_result
timeout.sleep()
return False
def verify_bgp_soo_on_route(
device,
address_family,
route,
soo_rt,
vrf=None,
rd=None,
max_time=60,
check_interval=15,
):
""" Verifies soo exists on route
Args:
Required:
device('obj'): device to verify on
address_family('str'): address_family to verify under
route('str'): route to verify
One or the other:
vrf('str'): verify using vrf
rd('str'): verify using rd
Optional:
timeout('obj'): timeout object to override default
Returns:
True
False
Raise:
None
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
if vrf:
result = get_bgp_route_ext_community(device,
address_family,
route,
vrf=vrf)
if not result:
return False
elif rd:
result = get_bgp_route_ext_community(device,
address_family,
route,
rd=rd)
if not result:
return False
if result and "SoO:{}".format(soo_rt) in result:
return True
timeout.sleep()
return False
def verify_ntp_association_with_server(
device, ip_address_server, peer_mode, max_stratum, max_time=15, check_interval=5
):
"""Verify association with server
Args:
server (`obj`): Server Device object
ip_address_server (`str`): IP address to server
peer_mode (`str`): peer mode type
max_stratum (`int`): maximum stratum value
max_time (int): Maximum wait time for the trigger,
in seconds. Default: 15
check_interval (int): Wait time between iterations when looping is needed,
in seconds. Default: 5
Returns:
True
False
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
result = False
try:
out = device.parse("show ntp associations detail")
except SchemaEmptyParserError as e:
log.error("Parser not found for command " "'show ntp associations detail'")
return False
for vrf in out["vrf"].keys():
try:
allowed_ip_dict = out["vrf"][vrf]["associations"]["address"][
ip_address_server
]["local_mode"]["client"]["isconfigured"]["True"]
stratum_found = allowed_ip_dict["stratum"]
refid = allowed_ip_dict["refid"]
peer_mode_found = allowed_ip_dict["peer"][refid]["local_mode"][
peer_mode
]["local_mode"]
except KeyError as e:
log.error(
"Could not find following key{}, exception {}".format(
peer_mode, str(e)
)
)
result = False
if stratum_found < max_stratum and peer_mode_found == peer_mode:
result = True
else:
result = False
if result:
return result
timeout.sleep()
return result
def verify_routing_neighbors_advertised_routes_exists(
device,
address_family,
neighbor,
vrf,
initial_route,
rd=None,
max_time=60,
check_interval=10,
output=None,
):
""" Verify if neighbors advertised routes exists
Args:
device ('obj'): Device object
address_family ('str'): address family
rd ('str'): rd export value
neighbor ('str'): neighbor address to find routes
vrf ('str'): vrf name
initial_route ('str'): intial route to search
max_time ('int') : max time for Timeout
check_interval ('int'): interval for Timeout
output ('list'): list of neighbors advertised routes (get_bgp_neighbors_advertised_routes)
Returns:
True
False
"""
result = False
timeout = Timeout(max_time=max_time, interval=check_interval)
while timeout.iterate():
if not output:
advertised_routes = list(
get_bgp_neighbors_advertised_routes(
device=device,
address_family=address_family,
neighbor_address=neighbor,
rd=rd,
vrf=vrf,
))
if not advertised_routes:
return False
else:
advertised_routes = output
if advertised_routes:
for route in advertised_routes:
if route == initial_route:
result = True
return result
else:
timeout.sleep()
return result
def verify_ospf_neighbor_state(device,
expected_state,
interface,
neighbor_address=None,
extensive=False,
max_time=60,
check_interval=10):
""" Verifies state of ospf neighbor
Args:
device ('obj'): device to use
expected_state ('str'): OSPF adjacency state that is expected
interface ('str'): Name of interface
neighbor_address ('str'): Neighbor address
extensive ('bool'): If ospf command is extensive
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
if extensive:
output = device.parse('show ospf neighbor extensive')
else:
output = device.parse('show ospf neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
#{'ospf-neighbor-information': {'ospf-neighbor': [{}]}}
neighbors = output.q.get_values('ospf-neighbor')
for neighbor in neighbors:
#'interface-name': 'ge-0/0/0.0'
#'ospf-neighbor-state': 'Full'
if not neighbor_address:
if neighbor.get('interface-name',[]) == interface and \
neighbor.get('ospf-neighbor-state',[]).lower() == expected_state.lower():
return True
else:
if neighbor.get('neighbor-address',[]) == neighbor_address and \
neighbor.get('ospf-neighbor-state',[]).lower() == expected_state.lower():
return True
timeout.sleep()
return False
def verify_interface_errors(device,
interface,
expected_value=None,
input=False,
output=False,
max_time=30,
check_interval=10):
""" Verify interface input and output errors
Args:
device (`obj`): Device object
interface (`str`): Pass interface in show command
expected_value (`int`, Optional): Expected errors values. Defaults to None
input (`bool`, Optional): True if input errors to verify. Default to False.
output (`bool`, Optional): True if output errors to verify. Default to False.
max_time (`int`, Optional): Max time, default: 60 seconds
check_interval (`int`, Optional): Check interval, default: 10 seconds
Returns:
result (`bool`): Verified result
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
cmd = 'show interface {interface}'.format(interface=interface)
out = device.parse(cmd)
except SchemaEmptyParserError:
timeout.sleep()
continue
interface, data = next(iter(out.items()))
data = Dq(data)
if input and output:
input_errors = data.get_values("in_errors", 0)
output_errors = data.get_values("out_errors", 0)
if input_errors == output_errors == expected_value:
return True
elif input:
input_errors = data.get_values("in_errors", 0)
if input_errors == expected_value:
return True
elif output:
output_errors = data.get_values("out_errors", 0)
if output_errors == expected_value:
return True
timeout.sleep()
continue
return False
def verify_ospf_advertising_router_metric_in_database(device,
lsa_id,
ospf_link_id,
expected_metric,
max_time=60,
check_interval=10):
""" Verifies ospf advertising router metric in database
Args:
device ('obj'): Device to use
lsa_id: lsa_id to check
ospf_link_id ('str'): Ospf link id to check
expected_metric ('str'): Metric of desired ospf link
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
cmd = 'show ospf database advertising-router {ipaddress} extensive'.format(
ipaddress=lsa_id)
out = device.parse(cmd)
except SchemaEmptyParserError:
timeout.sleep()
continue
# Example output:
# {
# "ospf-database-information": {
# "ospf-database": [
# {
# "ospf-router-lsa": {
# "ospf-link": [
# {
# "link-id": "106.187.14.240",
# "metric": "1"
# }]}}]}}
for ospf_database in out.q.get_values('ospf-database'):
ospf_links = Dq(ospf_database).get_values('ospf-link')
for ospf_link in ospf_links:
if ospf_link.get('link-id', None) == ospf_link_id \
and ospf_link.get('metric', None) == str(expected_metric):
return True
timeout.sleep()
return False
def verify_bgp_all_peer_state(device,
expected_state,
max_time=60,
check_interval=10):
"""
Verify bgp all peer state
Args:
device('obj'): device to use
expected_state('str') : Expected peer state
Default to `Establ`
max_time ('int', optional): Maximum time to keep checking. Default to 60
check_interval ('int', optional): How often to check. Default to 10
Returns:
Boolean
Raises:
N/A
"""
# show commands: "show bgp summary"
# {
# "bgp-information": {
# "bgp-peer": [
# {
# (snip)
# "flap-count": "0",
# "input-messages": "4",
# "output-messages": "4",
# "peer-address": "10.0.0.2",
# "peer-as": "65002",
# "peer-state": "Establ",
# "route-queue-count": "0"
# },
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show bgp summary')
except SchemaEmptyParserError:
timeout.sleep()
continue
special_case_state = re.compile(r'([\d\/]+\s+[\d\/]+)')
peer_state = [val if not special_case_state.match(val) else 'Establ' for val in out.q.get_values('peer-state')]
if set([expected_state]) == set(peer_state):
return True
timeout.sleep()
return False
def verify_bfd_session(device,
session_address,
expected_session_state=None,
expected_session_multiplier=None,
max_time=60,
check_interval=10,
expected_interface=None):
""" Verifiy the session state
Args:
device (obj): Device object
session_address (str): Session address
expected_session_state (str): Expected session state
expected_session_multiplier (str): Expected session multiplier
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
expected_interface (str, optional): Expected interface to check
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show bfd session')
except SchemaEmptyParserError:
timeout.sleep()
continue
sessions_ = out.q.get_values('bfd-session')
for session in sessions_:
if session.get('session-neighbor') == session_address:
if expected_session_multiplier and \
session.get('session-adaptive-multiplier') != str(expected_session_multiplier):
continue
if expected_session_state and \
session.get('session-state').lower() != expected_session_state.lower():
continue
if expected_interface and session.get(
'session-interface') != expected_interface:
continue
return True
timeout.sleep()
return False
def verify_lacp_interface_receive_state(
device,
interface,
expected_state,
expected_interface=None,
max_time=60,
check_interval=10,
):
"""Verify the state of an lackp interface
Args:
device (obj): Device object
interface (str): Interface name. Will be used if expected_interface isn't set
expected_state (str): Expected state to check against. Defaults to None.
expected_interface (str, optional): Expected interface to check against. Defaults to None.
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
"""
interface_to_use = expected_interface if expected_interface else interface
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse(
'show lacp interfaces {interface}'.format(interface=interface))
except SchemaEmptyParserError:
timeout.sleep()
continue
# {
# "lacp-interface-information-list": {
# "lacp-interface-information": {
# "lag-lacp-protocol": [
# {
# "lacp-mux-state": "Collecting " "distributing",
# "lacp-receive-state": "Current",
# "lacp-transmit-state": "Fast " "periodic",
# "name": "ge-0/0/0",
# },
interfaces = out.q.get_values('lag-lacp-protocol')
states = {
intr['name']: intr['lacp-receive-state']
for intr in interfaces
}
if expected_state == states.get(interface_to_use, None):
return True
timeout.sleep()
return False
def verify_bgp_holdtime(device,
expected_holdtime,
interface,
max_time=60,
check_interval=10):
"""
Verify bgp holdtimer with peer {interface}
Args:
device('obj'): device to use
interface('str'): Peer interface
expected_holdtime('str'): Expected holdtime
max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds
check_interval ('int', optional): How often to check. Default to 10 seconds
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
filter_output = out.q.contains('holdtime|peer-address',
regex=True).reconstruct()
# {'bgp-information': {
# 'bgp-peer': [
# {'bgp-option-information': {
# 'holdtime': '30'},
# 'peer-address': '20.0.0.3+179'},
# {'bgp-option-information': {
# 'holdtime': '10'},
# 'peer-address': '2001:20::3+179'}]}}
peer_list = filter_output.get('bgp-information').get('bgp-peer')
for peer in peer_list:
# 20.0.0.3+63208
peer_address = peer.get('peer-address').split('+')[0]
if peer_address == interface and\
Dq(peer).get_values('holdtime')[0] == str(expected_holdtime):
return True
timeout.sleep()
return False
def verify_bgp_peer_import_value(device: object,
peer_address: str,
expected_import_value: str,
max_time: int = 60,
check_interval: int = 10) -> bool:
"""Verifies BGP peer import value
Args:
device (object): Device object
peer_address (str): Peer address
expected_import_value (str): Expected import value to check against
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
Returns:
bool: True/False
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
# Example dict
# "bgp-information": {
# "bgp-peer": [
# {
# "bgp-option-information": {
# "import-policy": str,
for peer in out.q.get_values('bgp-peer'):
peer_interface_ = peer.get('peer-address')
# 20.0.0.3+63208
if '+' in peer_interface_:
peer_interface_ = peer_interface_.split('+')[0]
# 20.0.0.2/24
if '/' in peer_address:
peer_address = peer_address.split('/')[0]
if peer_interface_ != peer_address:
continue
if Dq(peer).get_values('import-policy',
0) == expected_import_value:
return True
timeout.sleep()
return False
def verify_ospf_metric(device,
interface,
metric,
max_time=60,
check_interval=10):
"""Verify the OSPF metric
Args:
device (obj): Device object
interface (str): Interface name
metric (str): OSPF metric
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show ospf interface extensive')
except SchemaEmptyParserError:
timeout.sleep()
continue
# Example dictionary
# "ospf-interface": [
# {
# "interface-name": "ge-0/0/0.0",
# "ospf-interface-topology": {
# "ospf-topology-metric": "5",
# },
# },
ospf_interface_list = Dq(out).get_values('ospf-interface')
for ospf_interface_dict in ospf_interface_list:
#{'interface-name': 'ge-0/0/1.0'}
interface_ = Dq(ospf_interface_dict).get_values(
'interface-name', 0)
#{'ospf-topology-metric': '5'}
metric_ = Dq(ospf_interface_dict).get_values(
'ospf-topology-metric', 0)
if interface_.lower() == interface.lower() and str(metric_) == str(
metric):
return True
timeout.sleep()
return False
def verify_smallest_stratum_ntp_system_peer(device, max_time=90, check_interval=15):
""" Verify NTP server with the smallest stratum is elected as system peer
Args:
device (`obj`): Device object
max_time (int): Maximum wait time for the trigger,
in seconds. Default: 90
check_interval (int): Wait time between iterations when looping is needed,
in seconds. Default: 15
Returns:
result (`bool`): Verified result
sys_peer (`str`): System peer ip
other_peers (`list`): Other peers ip
"""
result = ""
sys_peer = ""
other_peers = []
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = device.parse("show ntp associations")
stratum = (
out.get("clock_state", {}).get("system_status", {}).get("clock_stratum")
)
sys_peer = (
out.get("clock_state", {})
.get("system_status", {})
.get("associations_address")
)
for peer, data in out["peer"].items():
if peer != sys_peer:
other_peers.append(peer)
peer_stratum = data["local_mode"]["client"]["stratum"]
if stratum > peer_stratum:
log.info(
"Found peer {peer} has a smaller stratum {stratum}".format(
peer=peer, stratum=peer_stratum
)
)
result = False
break
else:
result = True
if not result:
timeout.sleep()
else:
break
return result, sys_peer, other_peers
def verify_module_status(device,
timeout=180,
interval=30,
ignore_modules=None):
''' Check status of slot using 'show platform'
Args:
device ('obj'): Device object
timeout ('int'): Max timeout to re-check slot status
interval ('int'): Max interval to re-check slot status
ignore_modules ('list'): Modules to ignore status check
'''
timeout = Timeout(max_time=timeout, interval=interval)
while timeout.iterate():
# Reset
failed_slots = []
try:
output = device.parse("show platform")
except SchemaEmptyParserError:
timeout.sleep()
continue
# Check state for all slots
failed_slots = Dq(output).contains('state').\
not_contains_key_value('state', 'empty').\
not_contains_key_value('state',
'.*ok.*|standby|ha-standby|Ready',
value_regex=True).\
get_values('slot')
# To ignore specified modules
if ignore_modules:
for module in ignore_modules:
if module in failed_slots:
failed_slots.remove(module)
log.info("Ignoring the following modules '{}'".\
format(ignore_modules))
if not failed_slots:
log.info("All modules on '{}' are in stable state".\
format(device.name))
break
else:
log.warning("The following modules are not in stable state {}".\
format(failed_slots))
log.warning(
"Sleeping {} seconds before rechecking".format(interval))
timeout.sleep()
continue
else:
raise Exception("Modules on '{}' are not in stable state".\
format(device.name))
def verify_specific_route(device,
address,
learn_protocol,
max_time=60,
check_interval=10):
"""Verifies address list agianst 'show route protocol {protocol}'
Args:
device ('obj'): device to use
address('str'): address to search for
learn_protocol('str'): Learned protocol
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
output = device.parse(
"show route {address}".format(address=address))
except SchemaEmptyParserError:
timeout.sleep()
continue
# [{'rt-entry': {'active-tag': '*', 'protocol-name': 'OSPF',
# 'preference': '10', 'metric': '1', 'age': {'#text': '00:00:01'},
# 'nh': [{'to': '106.187.14.158', 'via': 'ge-0/0/0.0'}]},
# 'rt-destination': '59.128.2.250/32'}]
rt_destination_ = Dq(output).get_values("rt-destination", 0)
# [{'rt-entry': {'active-tag': '*', 'protocol-name': 'OSPF',
# 'preference': '10', 'metric': '1', 'age': {'#text': '00:00:01'}
protocol_name = Dq(output).get_values("protocol-name", 0)
if protocol_name:
if rt_destination_.startswith(str(address)) and \
protocol_name.lower() == learn_protocol.lower():
return True
else:
timeout.sleep()
continue
timeout.sleep()
return False
def verify_bfd_session_destination_detail(
device,
address,
expected_session_state=None,
expected_received_parameters_state=None,
max_time=60,
check_interval=10,
):
""" Verifies bfd session destination detail
Args:
device ('obj'): device to use
address ('str'): IP address for command
expected_session_state ('str'): Session state to verify
expected_received_parameters_state ('str'): Received parameter state to verify
max_time ('int'): Max time to retry. Default to 60
check_interval ('int'): Number of check in interval. Default to 10
Returns:
True
False
Raises:
None
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse(
"show bfd session destination {address} detail".format(
address=address
)
)
except SchemaEmptyParserError:
timeout.sleep()
continue
if expected_session_state:
state = out.q.contains('session').get_values('state', 0)
if state != expected_session_state:
timeout.sleep()
continue
if expected_received_parameters_state:
received_parameters_state = out.q.contains(
'received_parameters').get_values('state', 0)
if received_parameters_state != expected_received_parameters_state:
timeout.sleep()
continue
timeout.sleep()
return False
def verify_device_tracking_counters_vlan_dropped(device,
vlanid,
feature,
protocol,
message,
num_dropped,
max_time=20,
check_interval=10):
""" Verify missing ipv6 source guard configurations
Args:
device('obj'): device object
vlanid('str'): vlan id
feature('str'): feature
protocol('str'): protocol
message('str'): message type
num_dropped('int'): number of dropped packets
max_time('int', optional): max check time. Defaults to 20
check_interval('int', optional): check intervals. Defaults to 10
Returns:
True
False
Raises:
None
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
output = device.parse(
'show device-tracking counters vlan {vlanid}'.format(
vlanid=vlanid))
dropped = output.get('vlanid', {}).get(int(vlanid), {}) \
.get('dropped', {}).get(feature, {})
if dropped:
if dropped['protocol'] == protocol and dropped['message'] == message and \
dropped['dropped'] == num_dropped:
return True
else:
log.info(
'Expected protocol: {protocol}, message: {message}, dropped: {dropped}. '
'Instead got protocol: {target_protocol}, message: {target_message}, '
'dropped: {target_dropped}.'.format(
protocol=protocol,
message=message,
dropped=num_dropped,
target_protocol=dropped['protocol'],
target_message=dropped['message'],
target_dropped=dropped['dropped']))
return False
timeout.sleep()
log.info('Dropped message type cannot be found')
return False
def verify_mpls_forwarding_table_has_prefix_in_subnet_range(
device, subnet, max_time=120, check_interval=30):
""" Verifies local label for entries with a prefix inside subnet
Args:
device ('obj'): Device to use
subnet ('str'): Subnet to verify inside
max_time ('int'): Max time to check
check_interval ('int'): How often to check
returns:
True/False
raises:
N/A
"""
log.info(
'Checking atleast one entry has a prefix in subnet {subnet} range'.
format(subnet=subnet))
try:
subnet = IPNetwork(subnet)
except Exception:
log.info('Bad subnet provided')
return False
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show mpls forwarding-table')
except SchemaEmptyParserError:
log.info('Parser output is empty')
timeout.sleep()
continue
for vrf in out.get('vrf'):
for local_label in out['vrf'][vrf].get('local_label'):
for out_label in out['vrf'][vrf]['local_label'][local_label]\
.get('outgoing_label_or_vc'):
for prefix in out['vrf'][vrf]['local_label'][local_label]\
['outgoing_label_or_vc'][out_label]\
.get('prefix_or_tunnel_id'):
try:
pfx = IPNetwork(prefix)
except Exception:
continue
if pfx in subnet:
return True
timeout.sleep()
return False
def execution(self, device, **kwargs):
# Init
status = OK
# create timeout object
timeout = Timeout(max_time=int(self.args.cpucheck_timeout),
interval=int(self.args.cpucheck_interval))
# loop status
loop_stat_ok = True
if not hasattr(self, 'PARSER_MODULE'):
return WARNING('Does not have CPU related parsers to check')
while timeout.iterate():
# Execute command to get five minutes usage percentage
try:
cpu_dict = self.PARSER_MODULE(device).parse(sort_time='5min',
key_word='CPU')
except Exception as e:
return ERRORED(
'No output from show processes cpu\n{}'.format(e))
# Check 5 minutes percentage smaller than cpucheck_fivemin_pcnt
if int(cpu_dict['five_min_cpu']) >= int(
self.args.cpucheck_fivemin_pcnt):
message = "****** Device {d} *****\n".format(d=device.name)
message += "Excessive CPU utilization detected for 5 min interval\n"
message += "Allowed: {e}%\n".format(
e=self.args.cpucheck_fivemin_pcnt)
message += "Measured: FiveMin: {r}%".format(
r=cpu_dict['five_min_cpu'])
loop_stat_ok = False
timeout.sleep()
else:
message = "***** CPU usage is Expected ***** \n"
message += "Allowed threashold: {e} \n"\
.format(e=self.args.cpucheck_fivemin_pcnt)
message += "Measured from device: {r}"\
.format(r=cpu_dict['five_min_cpu'])
loop_stat_ok = True
status += OK(message)
logger.info(banner(message))
break
if not loop_stat_ok:
status += CRITICAL(message)
logger.error(banner(message))
# Final status
return status
