def verify_segment_routing_traffic_eng_policies(
device,
policy_name=None,
expected_preference=None,
expected_admin_status=None,
expected_oper_status=None,
expected_metric_type=None,
expected_path_accumulated_metric=None,
expected_path_status=None,
expected_affinity_type=None,
expected_affinities=None,
expected_endpoint_ip=None,
max_time=30,
check_interval=10):
""" Verifies configured traffic_eng policies have expected configurations
Args:
device ('obj'): Device to use
policy_name ('str'): Policy name to verify. If not specified will verify all
expected_admin_status ('str'): Expected admin status
expected_oper_status ('str'): Expected operational status
expected_metric_type ('str'): Expected metric type
expected_path_accumulated_metric ('int'): Expected path accumulated metric
expected_path_status ('str'): Expected path status
expected_affinity_type ('str'): Expected affinity type
expected_affinities ('list'): Expected affinities
expected_preference ('int'): Expected preference path
expected_endpoint_ip ('str'): Expected Endpoint IP
max_time ('int'): Maximum amount of time to keep checking
check_interval ('int'): How often to check
Returns:
True/False
Raises:
N/A
"""
if (not expected_admin_status and not expected_oper_status
and not expected_metric_type
and not expected_path_accumulated_metric
and not expected_path_status and not expected_affinity_type
and not expected_affinities and not expected_endpoint_ip):
log.info('Must provide at-least one optional argument to verify')
return False
if policy_name:
cmd = 'show segment-routing traffic-eng policy name {policy}'.format(
policy=policy_name)
else:
cmd = 'show segment-routing traffic-eng policy all'
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse(cmd)
except SchemaEmptyParserError:
log.info('Parser output is empty')
timeout.sleep()
continue
for policy in out:
if expected_endpoint_ip:
endpoint_ip = out[policy].get('end_point', None)
if endpoint_ip != expected_endpoint_ip:
# Check other policy
continue
admin = out[policy].get('status', {}).get('admin', '')
if expected_admin_status and expected_admin_status != admin:
log.info(
'Expected admin status is "{admin_status}" actual is "{admin}"'
.format(admin_status=expected_admin_status, admin=admin))
break
operational = out[policy].get('status',
{}).get('operational',
{}).get('state', '')
if expected_oper_status and expected_oper_status != operational:
log.info(
'Expected operational status is "{operational_status}" actual is "{operational}"'
.format(operational_status=expected_oper_status,
operational=operational))
break
for preference in out[policy].get('candidate_paths',
{}).get('preference', {}):
if expected_preference and expected_preference != preference:
continue
if out[policy]['candidate_paths']['preference'][
preference].get('path_type'):
path_type_dict = out[policy]['candidate_paths'][
'preference'][preference]['path_type']
if 'dynamic' in path_type_dict:
metric = path_type_dict['dynamic'].get(
'metric_type', '')
if expected_metric_type and expected_metric_type != metric:
log.info(
'Expected metric type for path {path} is "{expected}" actual is "{actual}"'
.format(expected=expected_metric_type,
actual=metric,
path=preference))
break
status = path_type_dict['dynamic'].get('status', '')
if expected_path_status and expected_path_status != status:
log.info(
'Expected status for path {path} is "{expected}" actual is "{actual}"'
.format(expected=expected_path_status,
actual=status,
path=preference))
break
accumulated_metric = path_type_dict['dynamic'].get(
'path_accumulated_metric', '')
if (expected_path_accumulated_metric
and isinstance(accumulated_metric, int)
and expected_path_accumulated_metric !=
accumulated_metric):
log.info(
'Expected accumulated metric for path {path} is "{expected}" actual is "{actual}"'
.format(
expected=expected_path_accumulated_metric,
actual=accumulated_metric,
path=preference))
break
elif 'explicit' in path_type_dict:
for segment in path_type_dict['explicit'].get(
'segment_list', {}):
metric = path_type_dict['explicit'].get(
'segment_list',
{}).get(segment, {}).get('metric_type', '')
if expected_metric_type and expected_metric_type != metric:
log.info(
'Expected metric type for path {path} is "{expected}" actual is "{actual}"'
.format(expected=expected_metric_type,
actual=metric,
path=preference))
break
status = path_type_dict['explicit'].get(
'segment_list', {}).get(segment,
{}).get('status', '')
if expected_path_status and expected_path_status != status:
log.info(
'Expected path status for path {path} is "{expected}" actual is "{actual}"'
.format(expected=expected_path_status,
actual=status,
path=preference))
break
else:
continue
break
else:
log.info('Path type not defined in api call.')
break
if out[policy]['candidate_paths']['preference'][
preference].get('constraints'):
constraint_dict = out[policy]['candidate_paths'][
'preference'][preference]['constraints']
if expected_affinity_type and expected_affinity_type not in constraint_dict.get(
'affinity', {}):
log.info(
'Expected affinity_type is for path {path} "{expected}". Actual is "{actual}"'
.format(expected=expected_affinity_type,
actual=list(constraint_dict.keys()),
path=preference))
break
if expected_affinities:
for aff_type in constraint_dict.get('affinity', {}):
if not set(expected_affinities).issubset(
constraint_dict['affinity'][aff_type]):
log.info(
'Expected affinities "{expected}" for path {path} are not set'
.format(expected=expected_affinities,
path=preference))
break
else:
continue
break
else:
continue
break
else:
return True
timeout.sleep()
return False
def verify_ip_cef_nexthop_label(device,
ip,
expected_label=None,
vrf='default',
table=None,
max_time=30,
check_interval=10):
""" Verify ip cef nexthop does (not) have expected label
Args:
device (`obj`): Device object
ip (`str`): IP address
expected_label (`str`): Expected label. None if no label expected
vrf (`str`): Vrf name
table (`str`): Not used on IOSXE
max_time (`int`): Max time, default: 30
check_interval (`int`): Check interval, default: 10
Returns:
result (`bool`): Verified result
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
if vrf and vrf != 'default':
cmd = 'show ip cef vrf {vrf} {ip} detail'.format(vrf=vrf, ip=ip)
else:
cmd = 'show ip cef {} detail'.format(ip)
while timeout.iterate():
try:
out = device.parse(cmd)
except Exception as e:
log.error("Failed to parse '{}':\n{}".format(cmd, e))
timeout.sleep()
continue
reqs = R([
'vrf', vrf, 'address_family', '(.*)', 'prefix', '(.*)', 'nexthop',
'(.*)', 'outgoing_interface', '(?P<interface>.*)',
'outgoing_label', '(?P<outgoing_label>.*)'
])
found = find([out], reqs, filter_=False, all_keys=True)
if expected_label:
if found:
for item in found:
interface = item[1][-2]
label = ' '.join(item[0])
log.info(
"Found outgoing interface '{}' has outgoing label '{}', "
"expected to have label '{}'".format(
interface, label, expected_label))
if expected_label in label:
return True
else:
log.error("Failed to get outgoing label for '{}'".format(ip))
timeout.sleep()
continue
else:
if found:
log.error("Found outgoing label for '{}', "
"but expected no label".format(ip))
timeout.sleep()
continue
else:
log.info(
"No outgoing label aftar the nexthop info for '{}'".format(
ip))
return True
timeout.sleep()
return False
def verify_ping(device,
address=None,
ttl=None,
tos=None,
size=None,
wait=None,
mpls_rsvp=None,
loss_rate=0,
ping_size=None,
count=None,
source=None,
rapid=False,
do_not_fragment=False,
max_time=30,
check_interval=10):
""" Verify ping loss rate on ip address provided
Args:
device ('obj'): Device object
address ('str'): Address value
size {'str'}: Size value for ping command
tos {'str'}: tos value for ping command
ping_size {'str'}: data bytes expected
ttl ('int'): ttl value passed in command
wait ('int'): wait value passed in command
mpls_rsvp ('str'): MPLS RSVP value
loss_rate ('int'): Expected loss rate value
count ('int'): Count value for ping command
source ('str'): Source IP address, default: None
rapid ('bool'): Appears in command or not, default: False
do_not_fragment ('bool'): Appears in command or not, default: False
max_time (`int`): Max time, default: 30
check_interval (`int`): Check interval, default: 10
Returns:
Boolean
Raises:
None
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
if tos:
cmd = 'ping {address} source {source} size {size} count {count} tos {tos} rapid'.format(
address=address,
source=source,
size=size,
count=count,
tos=tos)
elif do_not_fragment and ping_size and count:
cmd = 'ping {address} size {ping_size} count {count} do-not-fragment'.format(
address=address,
ping_size=ping_size,
count=count,
)
elif address and count and source and rapid:
cmd = 'ping {address} source {source} count {count} rapid'.format(
address=address, source=source, count=count)
elif address and count and source:
cmd = 'ping {address} source {source} count {count}'.format(
address=address, source=source, count=count)
elif address and count and not ttl and not wait:
cmd = 'ping {address} count {count}'.format(address=address,
count=count)
elif address and count and ttl and wait:
cmd = 'ping {address} ttl {ttl} count {count} wait {wait}'.format(
address=address, ttl=ttl, count=count, wait=wait)
elif not address and mpls_rsvp:
cmd = 'ping mpls rsvp {rsvp}'.format(rsvp=mpls_rsvp)
elif address:
cmd = 'ping {address}'.format(address=address)
else:
log.info('Need to pass address as argument')
return False
try:
out = device.parse(cmd)
except SchemaEmptyParserError as e:
timeout.sleep()
continue
# Example dictionary structure:
# {
# "ping": {
# "address": "10.189.5.94",
# "data-bytes": 56,
# "result": [
# {
# "bytes": 64,
# "from": "10.189.5.94",
# "icmp-seq": 0,
# "time": "2.261",
# "ttl": 62
# },
# ],
# "source": "10.189.5.94",
# "statistics": {
# "loss-rate": 0,
# "received": 1,
# "round-trip": {
# "avg": "2.175",
# "max": "2.399",
# "min": "1.823",
# "stddev": "0.191"
# },
# "send": 1
# }
# }
# }
loss_rate_found = Dq(out).get_values("loss-rate", 0)
if size:
if loss_rate_found == loss_rate and Dq(out).get_values(
"data-bytes", 0) == int(size):
return True
if loss_rate_found == loss_rate:
return True
timeout.sleep()
return False
def verify_bgp_peer_prefixes_match(
device: object,
peer_address: str,
active: bool = True,
received: bool = True,
accepted: bool = True,
max_time: int = 60,
check_interval: int = 10
) -> bool:
"""Verifies whether BGP peer prefixes match or not
Args:
device (object): Device object
peer_address (str): Peer address
active (bool, optional): Check active prefix. Defaults to True.
received (bool, optional): Check received prefix. Defaults to True.
accepted (bool, optional): Check accepted prefix. Defaults to True.
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
Returns:
bool: True/False
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
# Example dict
# "bgp-information": {
# "bgp-peer": [
# {
# "bgp-rib": [
# {
# "accepted-prefix-count": str,
# "active-prefix-count": str,
# "received-prefix-count": str,
for peer in out.q.get_values('bgp-peer'):
peer_interface_ = peer.get('peer-address')
# 20.0.0.3+63208
if '+' in peer_interface_:
peer_interface_ = peer_interface_.split('+')[0]
# 20.0.0.2/24
if '/' in peer_address:
peer_address = peer_address.split('/')[0]
if peer_interface_ != peer_address:
continue
prefix_list_ = list()
if active:
prefix_list_.append(Dq(peer).get_values('active-prefix-count', 0))
if accepted:
prefix_list_.append(Dq(peer).get_values('accepted-prefix-count', 0))
if received:
prefix_list_.append(Dq(peer).get_values('received-prefix-count', 0))
if len(set(prefix_list_)) == 1:
return True
timeout.sleep()
return False
def verify_vpn_route_targets(
device,
route_targets,
rt_type,
address_family,
vrf=None,
route_distinguisher=None,
max_time=15,
check_interval=5,
):
""" Verify route target are imported, exported or both
Args:
device ('obj'): Device object
route_targets ('list'): list of route targets to check
ex.)
[
'65109:4005',
'65109:4006'
]
rt_type ('str'): route target type
ex.) rt_type = 'import' OR
rt_type = 'export' OR
rt_type = 'both'
address_family ('str'): address family to check
vrf ('str'): vrf name
route_distinguisher ('str'): route distinguisher value
max_time (int): Maximum wait time for the trigger,
in second. Default: 15
check_interval (int): Wait time between iterations when looping is needed,
in second. Default: 5
Returns:
True
False
"""
# Check if both route targets exists on device
if route_targets:
if not isinstance(route_targets, list):
log.error("route_targets must be list")
return False
result = True
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
result = True
for rt in route_targets:
current_rt = None
try:
current_rt = get_vrf_route_targets(
device=device,
rt_type=rt_type,
address_family=address_family,
vrf=vrf,
route_distinguisher=route_distinguisher,
)
except Exception as e:
log.error(str(e))
if not current_rt:
log.info(
"Route target of type {} not found for VRF {} on device {}"
.format(rt_type, vrf, device.name))
result = False
if result:
return result
timeout.sleep()
return result
def verify_bgp_error_message(device, interface, expected_message, expected_error_message, max_time=60, check_interval=10):
"""
Verify bgp last error
Args:
device('obj'): device to use
interface('str'): Peer interface
expected_message('str'): Expected message
expected_error_message('str') : Expected error message
max_time ('int', optional): Maximum time to keep checking. Default to 60
check_interval ('int', optional): How often to check. Default to 10
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# "flap-count": '0',
# "peer-address": '20.0.0.3+63208',
# "peer-restart-flags-received": 'Notification',
# "bgp-error": [
# {
# "name": "Hold Timer Expired " "Error",
# "receive-count": "40",
# "send-count": "27",
# },
# {"name": "Cease", "receive-count": "0", "send-count": "16"},
# ],
# .
# .
# .
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
for peer in peers_list:
peer_interface = peer.get('peer-address')
# 20.0.0.3+63208
if '+' in peer_interface:
peer_interface = peer_interface.split('+')[0]
notification_message = Dq(peer).get_values(
"peer-restart-flags-received", 0)
error_message_dict = Dq(peer).get_values("bgp-error", 0)
if len(notification_message) > 0 and error_message_dict:
if peer_interface == interface and notification_message == expected_message and error_message_dict.get('name') == expected_error_message:
return True
timeout.sleep()
return False
def verify_bgp_updown_time(device, given_seconds, invert=False, max_time=60, check_interval=10):
"""
Verify the up/down time of all neighbors is less than given_time
Args:
device('obj'): device to use
given_seconds('int') : Given time in seconds
max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds
check_interval ('int', optional): How often to check. Default to 10 seconds
Returns:
Boolean
Raises:
N/A
"""
op = operator.ge
if invert:
op = operator.lt
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp summary"
while timeout.iterate():
try:
out = device.parse('show bgp summary')
except SchemaEmptyParserError:
timeout.sleep()
continue
# 'bgp-peer': [{'bgp-rib': [{'accepted-prefix-count': '0',
# 'active-prefix-count': '0',
# 'name': 'inet.0',
# 'received-prefix-count': '0',
# 'suppressed-prefix-count': '0'}],
# 'elapsed-time': {'#text': '1:01'}, <-------------
# 'flap-count': '1',
# time_list: ['1:01', '57']
time_list = out.q.get_values('#text')
seconds_per_hour = 60*60
seconds_per_day = 24*seconds_per_hour
seconds_per_week = 7*seconds_per_day
for elapsed_time in time_list:
# '1:01'
p_min = re.compile(r'^\d+\:\d+$')
# '11:11:11'
p_hour = re.compile(r'^\d+\:\d+\:\d+$')
# "29w5d 22:42:36"
p_day = re.compile(r'^(?P<week>\d+)w(?P<day>\d+)d\s+(?P<hour>\d+)\:(?P<minute>\d+)\:(?P<second>\d+)$')
p_day_match = p_day.match(elapsed_time)
if p_min.match(elapsed_time):
min_sec_list = elapsed_time.split(':')
elapsed_time = int(min_sec_list[0])*60 + int(min_sec_list[1])
elif p_hour.match(elapsed_time):
hour_min_sec_list = elapsed_time.split(':')
elapsed_time = int(hour_min_sec_list[0])*seconds_per_hour+\
int(hour_min_sec_list[1])*60+\
int(hour_min_sec_list[2])
elif p_day_match:
group = p_day_match.groupdict()
elapsed_time = int(group['week'])*seconds_per_week+\
int(group['day'])*seconds_per_day+\
int(group['hour'])*seconds_per_hour+\
int(group['minute'])*60+\
int(group['second'])
else:
elapsed_time = int(elapsed_time)
# compare current up/dwn time with given time
if op(elapsed_time, given_seconds):
return False
timeout.sleep()
return True
def verify_no_ospf_neigbor_output(device,
expected_interface=None,
extensive=False,
max_time=60,
check_interval=10):
""" Verifies ospf neighbor doesn't exists
Args:
device ('obj'): device to use
expected_interface ('str'): Interface being searched for
extensive ('bool'): If ospf command is extensive
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
exists = False
while timeout.iterate():
if extensive:
try:
output = device.parse('show ospf neighbor extensive')
except SchemaEmptyParserError:
output = None
timeout.sleep()
continue
else:
try:
output = device.parse('show ospf neighbor')
except SchemaEmptyParserError:
output = None
timeout.sleep()
continue
for neighbor in Dq(output).get_values('ospf-neighbor'):
if neighbor.get('interface-name') == expected_interface:
exists = True
timeout.sleep()
break
else:
exists = False
timeout.sleep()
if not output or not exists:
return True
else:
return False
def verify_ospf_interface_cost(device,
interface,
expected_cost,
cost_type='ospf',
instance=None,
area=None,
max_time=60,
check_interval=15):
""" Verifies ospf cost on interface
Args:
device ('obj'): device to use
interface ('str'): Interface to use
cost_type ('str'): Cost type configured
expected_cost ('int'): Expected configured cost
instance ('str'): Instance to use
area ('str'): Area to use
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
if 'ospf' in cost_type.lower():
while timeout.iterate():
try:
out = device.parse(
'show ospf interface {interface} detail'.format(
interface=interface))
except SchemaEmptyParserError:
log.info('Parser is empty')
timeout.sleep()
continue
reqs = R([
'instance', '{}'.format(instance if instance else 'master'),
'areas', '{}'.format(area if area else '(.*)'), 'interfaces',
interface, 'cost', '(?P<cost>.*)'
])
found = find([out], reqs, filter_=False, all_keys=True)
if found:
keys = GroupKeys.group_keys(
reqs=reqs.args, ret_num={}, source=found, all_keys=True)
if 'cost' in keys[0] and int(expected_cost) == int(
keys[0]['cost']):
return True
timeout.sleep()
return False
elif 'te' in cost_type.lower():
while timeout.iterate():
try:
out = device.parse('show interfaces {interface} terse'.format(
interface=interface))
except SchemaEmptyParserError:
log.info('Parser is empty')
timeout.sleep()
continue
reqs = R([
interface, 'protocol', 'inet', '(.*)', 'local', '(?P<local>.*)'
])
found = find([out], reqs, filter_=False, all_keys=True)
if found:
keys = GroupKeys.group_keys(
reqs=reqs.args, ret_num={}, source=found, all_keys=True)
local_address = keys[0].get('local')
try:
out = device.parse('show ted database extensive')
except SchemaEmptyParserError:
log.info('Parser is empty')
timeout.sleep()
continue
reqs = R([
'node', '(.*)', 'protocol', '(.*)', 'to', '(.*)', 'local',
local_address.split('/')[0], 'remote', '(.*)', 'metric',
'(?P<metric>.*)'
])
found = find([out], reqs, filter_=False, all_keys=True)
if found:
keys = GroupKeys.group_keys(
reqs=reqs.args,
ret_num={},
source=found,
all_keys=True)
if 'metric' in keys[0] and int(expected_cost) == int(
keys[0]['metric']):
return True
timeout.sleep()
return False
log.info('This api does not support cost type {}'.format(cost_type))
def install_image(section, steps, device, images, save_system_config=False,
install_timeout=500, reload_timeout=800):
"""
Clean yaml file schema:
-----------------------
install_image:
images: <Image to install `list`> (Mandatory)
save_system_config: <Whether or not to save the system config if it was modified `bool`> (Optional) Default: False
install_timeout: <Maximum time to wait for install process to finish `int`> (Optional) Default: 500
reload_timeout: <Maximum time to wait for reload process to finish `int`> (Optional) Default: 800
Example:
--------
install_image:
images:
- /auto/some-location/that-this/image/stay-isr-image.bin
save_system_config: True
install_timeout: 1000
reload_timeout: 1000
"""
# Need to get directory that the image would be unpacked to
# This is the same directory the image is in
directory = images[0].split(':')[0]
# pacakges.conf is hardcoded because it is required to set
# the boot variable to packages.conf for 'install mode'
# ----------------------------------------------------------
# 'Bundle mode' is when we set the boot var to the image.bin
new_boot_var = directory+':packages.conf'
with steps.start("Configure boot variables for 'install mode' on '{dev}'".format(dev=device.hostname)) as step:
with step.start("Delete all boot variables") as substep:
# Get list of existing boot variables
try:
curr_boot_images = device.api.get_boot_variables(boot_var='current')
except Exception as e:
substep.failed("Unable to retrieve boot variables\nError: {e}".format(e=str(e)))
if not curr_boot_images:
substep.passed("No boot variables are configured")
# delete the existing boot variables
try:
device.api.execute_delete_boot_variable(
boot_images=curr_boot_images, timeout=60)
except Exception as e:
substep.failed("Failed to delete all boot variables\nError: {e}".format(e=str(e)))
else:
substep.passed("Deleted all boot variables")
with step.start("Configure boot system variable to '{boot_var}'".format(boot_var=new_boot_var)) as substep:
try:
device.api.execute_set_boot_variable(
boot_images=[new_boot_var], timeout=60)
except Exception as e:
substep.failed("Failed to configure new boot variables\nError: {e}".format(e=str(e)))
else:
substep.passed("Configured boot system variable")
with step.start("Copy running-configuration to startup-configuration") as substep:
try:
device.api.execute_copy_run_to_start(
max_time=60, check_interval=30)
except Exception as e:
substep.failed("Failed to copy running-config to startup-config\nError: {e}".format(e=str(e)))
else:
substep.passed("Copied running-config to startup-config")
# Verify next reload boot variables are correctly set
with step.start("Verify next reload boot variables are correctly set") as substep:
if not device.api.verify_boot_variable(boot_images=[new_boot_var]):
substep.failed("Boot variables are not correctly set to {}"
.format([new_boot_var]))
else:
substep.passed("Boot variables are correctly set".format(device.name))
with steps.start("Installing image '{img}' onto {dev}".format(
img=images[0], dev=device.hostname)) as step:
install_add_one_shot_dialog = Dialog([
Statement(pattern=r".*Press Quit\(q\) to exit, you may save "
r"configuration and re-enter the command\. "
r"\[y\/n\/q\]",
action='sendline(y)' if save_system_config else 'sendline(n)',
loop_continue=True,
continue_timer=False),
Statement(pattern=r".*Please confirm you have changed boot config "
r"to flash\:packages\.conf \[y\/n\]",
action='sendline(y)',
loop_continue=True,
continue_timer=False),
Statement(pattern=r".*This operation may require a reload of the "
r"system\. Do you want to proceed\? \[y\/n\]",
action='sendline(y)',
loop_continue=True,
continue_timer=False),
])
try:
device.execute('install add file {} activate commit'.format(images[0]),
reply=install_add_one_shot_dialog,
error_pattern=['.*FAILED: install_add_activate_commit.*'],
timeout=install_timeout)
except Exception as e:
step.failed("Installing image '{img}' failed. Error: {e}"
.format(img=images[0], e=str(e)))
with steps.start("Waiting for {dev} to reload".format(dev=device.hostname)) as step:
timeout = Timeout(reload_timeout, 60)
while timeout.iterate():
timeout.sleep()
device.destroy()
try:
device.connect(learn_hostname=True)
except Exception:
log.info("{dev} is not reloaded".format(dev=device.hostname))
else:
step.passed("{dev} has successfully reloaded".format(dev=device.hostname))
step.failed("{dev} failed to reboot".format(dev=device.hostname))
image_mapping = section.history['install_image'].parameters.setdefault(
'image_mapping', {})
image_mapping.update({images[0]: new_boot_var})
def _perform_issu(self, steps, upgrade_image, timeout=300):
"""Perform the ISSU steps in sequence on the ASR1K device:
1. Execute 'issu loadversion' to begin ISSU process
2. Poll until standby RP reaches 'ok' state
3. Verify ISSU state is now 'loadversion'
4. Execute 'issu runversion' to initiate RP failover
5. Reconnect to the device
6. Verify ISSU state is now 'runversion'
7. Execute 'issu acceptversion' to cancel rollback timer
8. Verify ISSU state is now 'acceptversion'
9. Verify ISSU rollback timer has been cancelled
10. Poll until standby RP reaches 'ok' state
11. Save running-configuration to startup-configuration
12. Execute 'issu commitversion' to complete ISSU process
13. Reload the device and then reconnect to it
14. Verify device is now booted with ISSU upgrade image
Raises:
Unicon errors
Exception
Example:
>>> _perform_issu(steps=steps, upgrade_image='someimage')
"""
# Init
device = self.device
lookup = Lookup.from_device(device)
filetransfer = FileUtils.from_device(device)
image_name = basename(upgrade_image)
# ======================================================================
# Get standby RP
# ======================================================================
with steps.start("Get standby RP information", continue_=True) as step:
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Standby RP
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if not ret:
raise Exception(
"Device '{}' does not have standby RP - cannot "
"perform ISSU".format(device.name))
standby_rp = ret[0][1][1]
srp = re.search('(?P<srp>(\d))', standby_rp).groupdict()['srp']
logger.info("Standby RP on '{dev}' is: '{standby_rp}'".format(
dev=device.name, standby_rp=standby_rp))
# ======================================================================
# issu loadversion
# ======================================================================
with steps.start("Execute 'issu loadversion' to begin ISSU process",
continue_=True) as step:
try:
output = device.execute('issu loadversion rp {srp} file '
'stby-harddisk:{image}'.format(
srp=srp, image=image_name),
timeout=600)
if 'FAILED' in output:
device.execute('issu abortversion', timeout=timeout)
raise Exception("Unable to execute 'issu loadversion'")
except Exception as e:
raise Exception("Unable to execute 'issu loadversion'")
# Poll until standby RP reaches 'ok' state in 'show platform'
logger.info("Poll until standby RP reaches 'ok' state")
platform_timeout = Timeout(max_time=1200, interval=120)
while platform_timeout.iterate():
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Create requirement to find standby-RP with 'ok, standby' state
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if ret:
logger.info("Stanby RP '{}' is in 'ok' state".\
format(standby_rp))
break
# Standby RP is not 'ok' state as yet, sleep and recheck
platform_timeout.sleep()
# Verify issu state
logger.info("Verify ISSU state is now 'loadversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='loadversion')
logger.info("ISSU state is 'loadversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# ======================================================================
# issu runversion
# ======================================================================
with steps.start("Execute 'issu runversion' to initiate RP failover",
continue_=True) as step:
try:
output = device.execute('issu runversion', timeout=timeout)
except SubCommandFailure:
# Timeout Unicon SubCommandFailure expected
# Wait a bit as the device is booting with the ISSU upgrade image
time.sleep(timeout)
pass
# Reconnect to device
logger.info("Reconnect to the device after runversion")
reconnect_timeout = Timeout(max_time=1200, interval=120)
self._reconnect(steps=steps, timeout=reconnect_timeout)
# Verify issu state
logger.info("Verify ISSU state is now 'runversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='runversion')
logger.info("ISSU state is 'runversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# ======================================================================
# issu acceptversion
# ======================================================================
with steps.start(
"Execute 'issu acceptversion' to cancel rollback timer",
continue_=True) as step:
try:
output = device.execute('issu acceptversion', timeout=timeout)
if 'FAILED' in output:
raise Exception("Unable to execute 'issu acceptversion'")
except Exception as e:
raise Exception("Unable to execute 'issu acceptversion'",
from_exception=e)
# Verify issu state
logger.info("Verify ISSU state is now 'acceptversion'")
try:
self.check_issu_state(device=device,
slot=standby_rp,
expected_state='acceptversion')
logger.info("ISSU state is 'acceptversion' as exepcted")
except Exception as e:
raise Exception(str(e))
# Verify rollback timer
logger.info("Verify ISSU rollback timer is now 'inactive'")
try:
self.check_issu_rollback_timer(device=device,
slot=standby_rp,
expected_state='inactive')
logger.info("ISSU rollback timer is 'inactive' as exepcted")
except Exception as e:
raise Exception(str(e))
# Poll until standby RP reaches 'ok' state in 'show platform'
logger.info("Poll until standby RP reaches 'ok' state")
platform_timeout = Timeout(max_time=1200, interval=120)
while platform_timeout.iterate():
platform_dict = lookup.parser.show_platform.\
ShowPlatform(device=device).parse()
# Create requirement to find standby-RP with 'ok, standby' state
rs = R([
'slot', '(?P<val1>.*)', 'rp', '(?P<val2>.*)', 'state',
'ok, standby'
])
ret = find([platform_dict], rs, filter_=False, all_keys=True)
if ret:
logger.info("Stanby RP '{}' is in 'ok' state".\
format(standby_rp))
break
# Standby RP is not 'ok' state as yet, sleep and recheck
platform_timeout.sleep()
# Save running-configuration to startup-configuration
logger.info("Save running-configuration to startup-configuration")
filetransfer.copyconfiguration(source='running-config',
destination='startup-config',
device=device)
# ======================================================================
# issu commitversion
# ======================================================================
with steps.start("Execute 'issu commitversion'",
continue_=True) as step:
try:
output = device.execute('issu commitversion', timeout=timeout)
if 'FAILED' in output:
raise Exception("Unable to execute 'issu commitversion'")
except Exception as e:
raise Exception("Unable to execute 'issu commitversion'",
from_exception=e)
# ======================================================================
# reload device
# ======================================================================
try:
reload_timeout = Timeout(max_time=1200, interval=120)
self.reload(steps=steps, timeout=reload_timeout)
except Exception as e:
raise Exception("Unable to reload the device after ISSU completed",
from_exception=e)
# ======================================================================
# verify image version
# ======================================================================
with steps.start(
"Verify device is loaded with upgraded image after ISSU",
continue_=True) as step:
try:
output = device.execute('show version | i image')
if image_name in output:
logger.info("ISSU upgrade image is successfully loaded on "
"the device '{}'".format(device.name))
except Exception as e:
raise Exception("Unable to execute 'show version'",
from_exception=e)
def fabric_clean(section,
steps,
device,
cleaning_timeout=90,
reload_timeout=800,
sleep_after_reload=60):
""" This stage will clean APIC controllers.
The stage will execute 'acidiag touch clean' and then reload the controller.
Stage Schema
------------
fabric_clean:
cleaning_timeout (int, optional): Max time for cleaning scripts to execute.
Defaults to 90.
reload_timeout (int, optional): Max time for reload. Defaults to 800.
sleep_after_reload (int, optional): Time in seconds to sleep after the
device completes reloading. Defaults to 60.
Example
-------
fabric_clean:
cleaning_timeout: 90
"""
with steps.start("Cleaning the device") as step:
result = device.api.execute_clean_controller_fabric(
max_time=cleaning_timeout)
if not result:
step.failed("Failed to clean the device")
else:
step.passed("Successfully cleaned the device")
with steps.start("Reloading '{dev}'".format(dev=device.name)):
reload_dialog = Dialog([
Statement(
pattern=
r".*This command will restart this device\, Proceed\? \[y\/N\].*",
action='sendline(y)')
])
device.sendline('acidiag reboot')
reload_dialog.process(device.spawn)
with steps.start(
"Waiting for {dev} to reload".format(dev=device.hostname)) as step:
timeout = Timeout(reload_timeout, 60)
while timeout.iterate():
timeout.sleep()
device.destroy()
try:
device.connect(learn_hostname=True)
except Exception:
log.info("{dev} is not reloaded".format(dev=device.hostname))
else:
step.passed("{dev} has successfully reloaded".format(
dev=device.hostname))
step.failed("{dev} failed to reboot".format(dev=device.hostname))
log.info("Sleeping for '{}' seconds for '{}' to stabilize.".format(
sleep_after_reload, device.name))
time.sleep(sleep_after_reload)
def verify_ospf_neighbor_address(device,
neighbor_address,
expected_state='Full',
max_time=90,
check_interval=10,
expected_failure=False):
""" Verifies ospf neighbors address
Args:
device ('obj'): device to use
max_time ('int'): Maximum time to keep checking
Default to 90 secs
check_interval ('int'): How often to check
Default to 10 secs
neighbor_address ('str'): neighbor_address
expected_state (`str`): expected neighbor state
Default to `Full`
expected_failure (`bool`): flag to make result opposite
Default to False
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show ospf neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
# example of out
# {
# "ospf-neighbor-information": {
# "ospf-neighbor": [ # <-----
# {
# "activity-timer": "32",
# "interface-name": "ge-0/0/0.0",
# "neighbor-address": "10.189.5.94",
# "neighbor-id": "10.189.5.253",
# "neighbor-priority": "128",
# "ospf-neighbor-state": "Full"
# },
ospf_neighbors = out.q.get_values('ospf-neighbor')
result = []
for neighbor in ospf_neighbors:
if (neighbor['neighbor-address'] == neighbor_address
and neighbor['ospf-neighbor-state'] == expected_state):
result.append(True)
else:
continue
if expected_failure:
if False == all(result) or result == []:
return True
else:
if True == all(result):
return True
timeout.sleep()
return False
def verify_ospf_database(device,
lsa_type=None,
expected_lsa_id=None,
max_time=60,
check_interval=10,
extensive=True,
invert=False):
""" Verifies information from show ospf database
Args:
device ([obj]): Device object
lsa_type ([str], optional): LSA type to check. Defaults to None.
expected_lsa_id ([str], optional): Expected LSA ID to find. Defaults to None.
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
extensive (bool, optional): Extensive or not. Default to True.
invert (bool, optional): Inverts verification to check if criteria doesn't exist
"""
op = operator.ne
if invert:
op = operator.eq
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
if extensive:
if lsa_type:
out = device.parse(
'show ospf database {lsa_type} extensive'.format(
lsa_type=lsa_type.lower()))
else:
out = device.parse('show ospf database extensive')
else:
if lsa_type:
out = device.parse('show ospf database {lsa_type}'.format(
lsa_type=lsa_type.lower()))
else:
out = device.parse('show ospf database')
except SchemaEmptyParserError:
timeout.sleep()
continue
count = 0
ospf_database_ = out.q.get_values('ospf-database')
for database in ospf_database_:
if expected_lsa_id and op(
expected_lsa_id.split('/')[0], database.get('lsa-id')):
continue
# Add criteria to check against
count += 1
if not invert:
return True
else:
if count == len(ospf_database_):
return True
timeout.sleep()
return False
def check_xe_sanity_device_ready(self,
testbed,
steps,
max_time=60,
interval=10):
"""Check redudancy status, critial processes status and chassis properties
Args:
testbed (`obj`): Testbed object
steps (`obj`): aetest steps object
Returns:
None
Raises:
pyATS Results
"""
log.info(
banner('Check redudancy status,\n'
'critial processes status,\n'
'chassis properties'))
# get uut
devices = testbed.find_devices(alias='uut')
for uut in devices:
lookup = Lookup.from_device(uut)
# get platform pts
platform_pts = self.parameters.get('pts', {}).get('platform',
{}).get('uut', None)
# check redudancy
with steps.start("Perform Redundancy Check on device {} - "
"to check if device reach 'SSO' status".format(
uut.name)) as step:
# create timeout object
timeout = Timeout(max_time=int(max_time), interval=int(interval))
try:
lookup.sdk.libs.abstracted_libs\
.subsection.stack_ha_redundancy_state(
device=uut, timeout=timeout, platform_pts=platform_pts)
except Exception as e:
step.passx('Redundancy state SSO not reached in the stack',
from_exception=e)
if hasattr(uut, 'workers'):
with uut.allocate() as worker:
# check Process
with steps.start("Verify that all critical process are up "
"and running on device {}".format(
uut.name)) as step:
try:
lookup.sdk.libs.abstracted_libs\
.subsection.process_check(device=worker)
except Exception as e:
step.passx('Processes verification test failed')
# check Chasfs
with steps.start("Verify that Chasfs properties are updated "
"by Stack Manager on device {}".format(
uut.name)) as step:
try:
lookup.sdk.libs.abstracted_libs\
.subsection.chasfs_properties(device=worker, timeout=timeout)
except Exception as e:
step.passx(
'Chasfs verification test failed\n{}'.format(e))
else:
# check Process
with steps.start("Verify that all critical process are up "
"and running on device {}".format(
uut.name)) as step:
try:
lookup.sdk.libs.abstracted_libs\
.subsection.process_check(device=uut)
except Exception as e:
step.passx('Processes verification test failed')
# check Chasfs
with steps.start("Verify that Chasfs properties are updated "
"by Stack Manager on device {}".format(
uut.name)) as step:
try:
lookup.sdk.libs.abstracted_libs\
.subsection.chasfs_properties(device=uut, timeout=timeout)
except Exception as e:
step.passx('Chasfs verification test failed\n{}'.format(e))
def verify_ospf_interface(device,
expected_interface=None,
expected_interface_type=None,
expected_state=None,
extensive=True,
max_time=60,
check_interval=10,
expected_hello_interval=None):
""" Verifies ospf interface exists with criteria
Args:
device ('obj'): device to use
expected_interface ('str'): Interface to use
expected_interface_type ('str'): Interface type
expected_state ('str'): Interface state
extensive ('boolean'): Flag for extensive command
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
expected_hello_interval ('str'): Expected hello interval
Returns:
True/False
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
if extensive:
out = device.parse('show ospf interface extensive')
else:
out = device.parse('show ospf interface')
except SchemaEmptyParserError:
timeout.sleep()
continue
# {
# "ospf-interface-information": {
# "ospf-interface": [
# {
# "interface-name": "ge-0/0/0.0",
# "interface-type": "P2P",
# "ospf-interface-state": "PtToPt",
# "hello-interval": "100",
# },
for ospf_interface in out.q.get_values('ospf-interface'):
# check variables
intf = ospf_interface.get('interface-name', None)
if expected_interface and expected_interface != intf:
continue
intf_type = ospf_interface.get('interface-type', None)
if expected_interface_type and expected_interface_type != intf_type:
continue
intf_state = ospf_interface.get('ospf-interface-state', None)
if expected_state and expected_state != intf_state:
continue
intf_hello_interval = ospf_interface.get('hello-interval', None)
if expected_hello_interval and str(expected_hello_interval) != intf_hello_interval:
continue
return True
timeout.sleep()
return False
def verify_bgp_peer_state(device, interface, expected_state,
max_time=60, check_interval=10):
"""
Verify bgp peer state
Args:
device('obj'): device to use
interface('str'): Peer interface
expected_state('str') : Expected peer state
check_peer('boolean'): pass True if want to check peer address; default False
max_time ('int', optional): Maximum time to keep checking. Default to 60
check_interval ('int', optional): How often to check. Default to 10
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# 'flap-count': '0',
# 'peer-address': '20.0.0.3+63208',
# 'peer-state': 'Established',
# .
# .
# .
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
is_bgp_running = out.q.get_values("is-bgp-running")
if False in is_bgp_running:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
for peer in peers_list:
peer_interface = peer.get('peer-address')
peer_state = peer.get("peer-state")
# 20.0.0.3+63208
if '+' in peer_interface:
peer_interface = peer_interface.split('+')[0]
# 20.0.0.2/24
if '/' in interface:
interface = interface.split('/')[0]
if peer_interface == interface and peer_state == expected_state:
return True
timeout.sleep()
return False
def verify_ospf_neighbor_number(device,
expected_interface=None,
expected_number=None,
expected_state=None,
extensive=False,
max_time=60,
check_interval=10):
""" Verifies the number of ospf neighbors that meets the criteria
Args:
device ('obj'): device to use
expected_interface ('str'): Interface to use
expected_number ('str'): State occurrence
expected_state ('str'): Interface state
extensive ('bool'): Flag to differentiate show commands
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
if not extensive:
out = device.parse("show ospf neighbor")
else:
out = device.parse("show ospf neighbor extensive")
except SchemaEmptyParserError:
timeout.sleep()
continue
# {
# "ospf3-neighbor-information": {
# "ospf3-neighbor": [
# {
# "interface-name": "ge-0/0/0.0",
# "ospf-neighbor-state": "Full"
# },
count = 0
for neighbor in out.q.get_values('ospf-neighbor'):
# check variables
interface_name = neighbor.get('interface-name', None)
if expected_interface and expected_interface != interface_name:
continue
neighbor_state = neighbor.get('ospf-neighbor-state', None)
if expected_state and expected_state.lower(
) != neighbor_state.lower():
continue
# if all variables exist, count plus 1
count += 1
if count >= expected_number:
return True
timeout.sleep()
return False
def is_bgp_neighbor_authentication_key_configured(device, interface=None, max_time=60, check_interval=10):
"""
Verify that all of bgp neighbors have Authentication key is configured
Args:
device('obj'): device to use
interface('str'): peer interface. Default to None.
max_time ('int', optional): Maximum time to keep checking. Default to 60
check_interval ('int', optional): How often to check. Default to 10
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
auth_key_not_configured = False
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# 'flap-count': '0',
# 'peer-address': '20.0.0.3+63208',
# 'peer-state': 'Established',
# "bgp-option-information": {
# "address-families": "inet-unicast " "inet-labeled-unicast",
# "authentication-configured": True,
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
if interface:
# 20.0.0.3+63208
# 20.0.0.2/24
if '+' in interface:
interface = interface.split('+')[0]
elif '/' in interface:
interface = interface.split('/')[0]
for peer in peers_list:
# If interface is given, API checks authorization key is configured for given
# the interface else it will check the authorization key for all peers.
if interface:
peer_interface = peer.get('peer-address', '')
# 20.0.0.3+63208
# 20.0.0.2/24
if '+' in peer_interface:
peer_interface = peer_interface.split('+')[0]
elif '/' in peer_interface:
peer_interface = peer_interface.split('/')[0]
if peer_interface == interface and peer.get('bgp-option-information', {}).get(
'authentication-configured', False):
return True
elif not peer.get('bgp-option-information', {}).get(
'authentication-configured', False):
auth_key_not_configured = True
break
if not interface and not auth_key_not_configured:
return True
timeout.sleep()
return False
def verify_ospf_interface_in_database(device,
expected_interface,
expected_interface_type=None,
subnet_mask=None,
expected_metric=None,
adv_router=False,
max_time=60,
check_interval=10):
""" Verifies ospf interface exists with criteria
Args:
device ('obj'): device to use
expected_interface ('str'): Interface to use
expected_interface_type ('str'): Interface type
subnet_mask ('str'): Subnet mask
expected_metric ('str'): Metric of Interface
adv_router ('bool'): Whether to look for address in adversiting router
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show ospf database extensive')
except SchemaEmptyParserError:
timeout.sleep()
continue
#'ospf-database':[
# {'lsa-type':
# 'Router',
# 'lsa-id': '1.1.1.1'
# }
#]
for ospf_database in Dq(out).get_values('ospf-database'):
#'ospf-external-lsa':
# {'address-mask': '255.255.255.255',
# 'ospf-external-lsa-topology': {
# 'ospf-topology-name':
# 'default'}}
ospf_external_lsa = ospf_database.get('ospf-external-lsa',{})
if not adv_router:
if subnet_mask != None:
if 'address-mask' not in ospf_external_lsa:
continue
else:
#{'address-mask': '255.255.255.255'}
current_mask = IPAddress(ospf_external_lsa.get('address-mask')).netmask_bits()
if str(current_mask) != subnet_mask:
continue
#'type-value': '2'
if not ospf_external_lsa:
continue
if not ospf_external_lsa.get('ospf-external-lsa-topology',{}):
continue
if str(expected_metric) != ospf_external_lsa.get('ospf-external-lsa-topology',{}).get('type-value',{}):
continue
if expected_interface_type != None:
#'lsa-type': 'Extern'
lsa_type = ospf_database.get('lsa-type', None)
lsa_type = lsa_type.lower() if lsa_type else lsa_type
if expected_interface_type.lower() != lsa_type:
continue
#'lsa-id': '11.11.11.11'
lsa_id = ospf_database.get('lsa-id', None)
if expected_interface != lsa_id:
continue
else:
advertising_router = ospf_database.get('advertising-router', None)
if expected_interface != advertising_router:
continue
return True
timeout.sleep()
return False
def get_bgp_neighbor_prefixes_count(device, interface, max_time=60, check_interval=10):
"""
Get bgp neighbor accepted, received or advertised prefixes count
Args:
device('obj'): device to use
interface('str'): peer interface. Default to None.
max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds.
check_interval ('int', optional): How often to check. Default to 10 seconds.
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
prefixes_count_dict = {}
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# "bgp-rib": [
# {
# "accepted-prefix-count": "684",
# "active-prefix-count": "682",
# "advertised-prefix-count": "0",
# "bgp-rib-state": "BGP restart " "is complete",
# "name": "inet.0",
# "received-prefix-count": "684",
# "rib-bit": "20000",
# "send-state": "in sync",
# "suppressed-prefix-count": "0",
# },
while timeout.iterate():
try:
out = device.parse('show bgp neighbor')
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
# 20.0.0.3+63208
# 20.0.0.2/24
if '+' in interface:
interface = interface.split('+')[0]
elif '/' in interface:
interface = interface.split('/')[0]
for peer in peers_list:
peer_interface = peer.get('peer-address', '')
# 20.0.0.3+63208
# 20.0.0.2/24
if '+' in peer_interface:
peer_interface = peer_interface.split('+')[0]
elif '/' in peer_interface:
peer_interface = peer_interface.split('/')[0]
if peer_interface == interface:
bgp_rib_list = peer.get('bgp-rib', [])
for bgp_rib in bgp_rib_list:
if bgp_rib.get('accepted-prefix-count', None):
prefixes_count_dict['accepted_prefix_count'] = int(bgp_rib.get('accepted-prefix-count'))
if bgp_rib.get('received-prefix-count', None):
prefixes_count_dict['received_prefix_count'] = int(bgp_rib.get('received-prefix-count'))
if bgp_rib.get('advertised-prefix-count', None):
prefixes_count_dict['advertised_prefix_count'] = int(bgp_rib.get('advertised-prefix-count'))
return prefixes_count_dict
timeout.sleep()
return prefixes_count_dict
def verify_diff_timestamp(device, expected_spf_delay=None, ospf_trace_log=None,\
max_time=60, check_interval=10):
"""
Verify the difference between time on two logs
Args:
device('obj'): device to use
expected_spf_delay('float'): SPF change value
ospf_trace_log('str') : OSPF trace log
max_time ('int'): Maximum time to keep checking
check_interval ('int'): How often to check
Returns:
Boolean
Raises:
N/A
"""
timeout = Timeout(max_time, check_interval)
# show commands: "show log {ospf_trace_log}"
while timeout.iterate():
try:
output = device.parse('show log {ospf_trace_log}'.format(
ospf_trace_log=ospf_trace_log))
except SchemaEmptyParserError:
timeout.sleep()
continue
# Example parsed output:
#
# {
# "file-content": [
# " show log messages",
# " Mar 5 00:45:00 sr_hktGCS001 newsyslog[89037]: "
# "logfile turned over due to size>1024K",
# " Mar 5 02:42:53 sr_hktGCS001 sshd[87374]: Received "
# "disconnect from 10.1.0.1 port 46480:11: disconnected by user",
# " Mar 5 02:42:53 sr_hktGCS001 sshd[87374]: "
# "Disconnected from 10.1.0.1 port 46480",
# " Mar 5 02:42:53 sr_hktGCS001 inetd[6841]: "
# "/usr/sbin/sshd[87371]: exited, status 255",
# }
file_content_list = output['file-content']
scheduled_time = start_time = datetime.datetime.now()
for i in file_content_list:
scheduled_time_str = device.api.get_ospf_spf_scheduled_time(i)
if scheduled_time_str:
scheduled_time = datetime.datetime.strptime(
scheduled_time_str, '%H:%M:%S.%f')
start_time_str = device.api.get_ospf_spf_start_time(i)
if start_time_str:
start_time = datetime.datetime.strptime(
start_time_str, '%H:%M:%S.%f')
time_change = (start_time - scheduled_time).seconds
if time_change == expected_spf_delay:
return True
timeout.sleep()
return False
def verify_bgp_peer_option(device,
interface,
protocol,
expected_bgp_option,
invert=False,
max_time=60,
check_interval=10):
"""
Verify bgp peer's bgp option
Args:
device('obj'): device to use
interface('str'): Peer interface
protocol('str'): protocol name
expected_bgp_option('str') : Expected peer bgp-option flag
invert (bool, optional): True if output does not contain expected_bgp_option. Defaults to False.
max_time ('int', optional): Maximum time to keep checking. Default to 60 seconds.
check_interval ('int', optional): How often to check. Default to 10 seconds.
Returns:
Boolean
Raises:
N/A
"""
op = operator.contains
if invert:
op = lambda data, check: operator.not_(operator.contains(data, check))
timeout = Timeout(max_time, check_interval)
# show commands: "show bgp neighbor"
# {'bgp-information': {'bgp-peer': [{
# 'peer-address': '20.0.0.3+63208',
# 'peer-state': 'Established',
# {'bgp-option-information': {
# "bgp-options": "Confed",
# .
# 20.0.0.2/24
interface = interface.split('/')[0]
while timeout.iterate():
try:
out = device.parse('show {protocol} neighbor'.format(protocol=protocol))
except SchemaEmptyParserError:
timeout.sleep()
continue
peers_list = out.q.get_values("bgp-peer")
for peer in peers_list:
peer_interface = peer.get('peer-address')
peer_bgp_option = Dq(peer).get_values('bgp-options')
# 20.0.0.3+63208
peer_interface = peer_interface.split('+')[0]
if peer_interface == interface and op(''.join(peer_bgp_option), expected_bgp_option):
return True
timeout.sleep()
return False
def get_cef_internal_repair_interface(device,
prefix,
vrf=None,
max_time=60,
check_interval=15):
""" Get cef internal output repair interface
Args:
device (`obj`): Device object
vrf (`str`): VRF to check
prefix (`str`): Prefix to check
max_time (`int`): Maximum time to keep checking
check_interval (`int`): How long to wait between checks
Raises:
N/A
Returns:
interface name/None
"""
if vrf:
cmd = 'show ip cef vrf {vrf} {prefix} internal'.format(vrf=vrf,
prefix=prefix)
else:
cmd = 'show ip cef vrf {prefix} internal'.format(prefix=prefix)
vrf = vrf if vrf else "default"
reqs = R([
"vrf",
"(?P<vrf>{vrf})".format(vrf=vrf),
"address_family",
"(?P<address_family>.*)",
"prefix",
"(?P<prefix>.*)",
"output_chain",
"tag_midchain",
"(?P<tag_midchain>.*)",
"frr",
"primary",
"repair",
"tag_adj",
"(?P<interface>.*)",
])
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse(cmd)
except SchemaEmptyParserError:
log.info("Parser output is empty")
timeout.sleep()
continue
found = find([out], reqs, filter_=False, all_keys=True)
if found:
for interface in found[0][0]:
return interface
timeout.sleep()
log.error("Failed to get repair interface")
return None
def verify_rib_fib_lfib_consistency(device,
route,
none_pattern='',
max_time=30,
check_interval=10):
""" Verify the outgoing label for route are the same in:
- show ip route <route>
- show ip cef <route>
- show mpls forwarding-table <route>
Args:
device (`obj`): Device object
route (`str`): Route or ipn
none_pattern (`list`): None label pattern
max_time (`int`): Max time, default: 30
check_interval (`int`): Check interval, default: 10
Returns:
result (`bool`): Verified result
"""
route = route.split('/')
if len(route) > 1:
route = '{} {}'.format(route[0], device.api.int_to_mask(int(route[1])))
else:
route = route[0]
patterns = ['No Label', 'implicit-null', 'Pop Label', 'none', '']
if none_pattern:
patterns.extend(none_pattern)
cmd1 = "show ip route {}".format(route)
cmd2 = "show ip cef {}".format(route)
cmd3 = "show mpls forwarding-table {}".format(route)
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
result = True
table = PrettyTable()
table.field_names = [
'RIB (INTF)', 'RIB (NH)', 'RIB (LABEL)', 'FIB (INTF)', 'FIB(NH) ',
'FIB (LABEL)', 'FIB (LOCAL LABEL)', 'LFIB (INTF)', 'LFIB(NH)',
'LFIB (LABEL)', 'LFIB (LOCAL LABEL)', 'PASS/FAIL'
]
try:
out1 = device.parse(cmd1)
except Exception as e:
log.error("Failed to parse '{}':\n{}".format(cmd1, e))
result = False
timeout.sleep()
continue
try:
out2 = device.parse(cmd2)
except Exception as e:
log.error("Failed to parse '{}':\n{}".format(cmd2, e))
result = False
timeout.sleep()
continue
try:
out3 = device.parse(cmd3)
except Exception as e:
log.error("Failed to parse '{}':\n{}".format(cmd3, e))
result = False
timeout.sleep()
continue
reqs1 = R(['entry', '(.*)', 'paths', '(.*)', '(?P<route>.*)'])
found1 = find([out1], reqs1, filter_=False, all_keys=True)
route_dict1 = {}
# eg: {"GigabitEthernet2": {
# "nexthop": "10.0.0.5",
# "from": "10.16.2.2",
# "age": "2w0d",
# "interface": "GigabitEthernet2",
# "prefer_non_rib_labels": true,
# "merge_labels": true,
# "metric": "3",
# "share_count": "1",
# "mpls_label": "16002",
# "mpls_flags": "NSF"}}
if found1:
for item in found1:
route_dict1.update({item[0]['interface']: item[0]})
else:
log.error("Failed to find route from '{}'".format(cmd1))
result = False
timeout.sleep()
continue
reqs2 = R([
'vrf', '(.*)', 'address_family', '(.*)', 'prefix', '(.*)',
'nexthop', '(?P<nexthop>.*)', 'outgoing_interface',
'(?P<interface>.*)', '(?P<sub>.*)'
])
found2 = find([out2], reqs2, filter_=False, all_keys=True)
route_dict2 = {}
# eg: {'GigabitEthernet2': {
# 'local_label': 16002,
# 'outgoing_label': ['16002'],
# 'nexthop': '10.0.0.5'}}
if found2:
for item in found2:
interface = item[1][-1]
nexthop = item[1][-3]
item[0].update({'nexthop': nexthop})
route_dict2.update({interface: item[0]})
else:
log.error(
"Failed to find outgoing interface from '{}'".format(cmd2))
result = False
timeout.sleep()
continue
reqs3 = R([
'vrf', '(.*)', 'local_label', '(?P<local_label>.*)',
'outgoing_label_or_vc', '(?P<outgoing_label>.*)',
'prefix_or_tunnel_id', '(?P<prefix>.*)', 'outgoing_interface',
'(?P<interface>.*)', 'next_hop', '(?P<next_hop>.*)'
])
found3 = find([out3], reqs3, filter_=False, all_keys=True)
route_dict3 = {}
# eg: {'GigabitEthernet4': {
# "local_label": 16,
# "outgoing_label": "Pop Label",
# "prefix": "10.0.0.13-A",
# "interface": "GigabitEthernet4",
# "next_hop": "10.0.0.13"}}
if found3:
keys = GroupKeys.group_keys(reqs=reqs3.args,
ret_num={},
source=found3,
all_keys=True)
for item in keys:
route_dict3.update({item['interface']: item})
else:
log.error(
"Failed to get outgoing interface from '{}'".format(cmd3))
result = False
timeout.sleep()
continue
if len(route_dict1) != len(route_dict2) != len(route_dict3):
log.error("The number of routes are different in the 3 output")
result = False
timeout.sleep()
continue
for interface in route_dict1.keys():
# get info from show ip route
rib_intf = interface
rib_nh = route_dict1[interface].get('nexthop', '')
rib_label = route_dict1[interface].get('mpls_label', '')
tmp_rib_label = None if rib_label in patterns else rib_label
# get info from show ip cef
fib_intf = interface if route_dict2.get(interface, '') else ''
fib_nh = route_dict2.get(interface, {}).get('nexthop', '')
fib_label = ' '.join(
route_dict2.get(interface, {}).get('outgoing_label', []))
fib_local = route_dict2.get(interface, {}).get('local_label', '')
tmp_fib_label = None if fib_label in patterns else fib_label
# get info from show mpls forwarding table
lfib_intf = interface if route_dict3.get(interface, '') else ''
lfib_nh = route_dict3.get(interface, {}).get('next_hop', '')
lfib_label = route_dict3.get(interface,
{}).get('outgoing_label', '')
lfib_local = route_dict3.get(interface, {}).get('local_label', '')
tmp_lfib_label = None if lfib_label in patterns else lfib_label
# if multiple entried forwarding table and prefer not rib labels, only check rib and lfib lable
# other wise check all labels
if (len(found3)>1 and route_dict1[interface].get('prefer_non_rib_labels') and (
tmp_rib_label == tmp_lfib_label and
rib_intf == fib_intf == lfib_intf and
rib_nh == fib_nh == lfib_nh)) \
or (rib_intf == fib_intf == lfib_intf and
rib_nh == fib_nh == lfib_nh and
tmp_rib_label == tmp_fib_label == tmp_lfib_label):
status = 'PASS'
else:
result = False
status = 'FAIL'
table.add_row([
rib_intf, rib_nh, rib_label, fib_intf, fib_nh, fib_label,
fib_local, lfib_intf, lfib_nh, lfib_label, lfib_local, status
])
log.info("Summary Result for {}:\n{}".format(route, table))
if result is True:
return result
timeout.sleep()
return result
def _loop_iterator(self,
steps,
testbed,
section,
name,
ret_list,
loop_variable_name=None,
loop_until=False,
max_time=None,
check_interval=None,
until=None,
do_until=None,
actions=None,
iterator_item=None,
every_seconds=None,
parallel=None,
**kwargs):
"""actually iterate over the actions under loop and call them"""
# TODO cant save vars in loop, enhancement needed
# timeout if until or do_until or loop_until are looping through infinity
timeout = Timeout(max_time, check_interval)
iterator_len = None
iterator_index = 0
iterator_len = max(len(itm)
for itm in iterator_item) if iterator_item else None
pcall_payload = []
# until condition would be sent to blitz_control
# in order to evaluate and see if the condition is true or not
# loop wont start if the condition is true with step as passed
until_condition = False
if until and blitz_control(self, section, until, 'until'):
until_condition = True
log.info('Until condition is met. Loop terminated')
while not until_condition:
# capturing the start time of the action to calculate
# the duration of the action
loop_start_time = datetime.now()
log.debug('loop start time: {}'.format(str(loop_start_time)))
if _check_pre_iteration(iterator_len, iterator_index, max_time,
timeout, parallel):
break
# assign each item in an iterable_item to a variable
# to be used later on within the actions
# the loop_variable_name should exist in the testcase
_save_iterator_items(self, section, loop_variable_name, iterator_item,
iterator_index)
# running all the actions and adding the outputs of those actions to return list
kwargs = {
'self': self,
'steps': steps,
'testbed': testbed,
'section': section,
'data': actions,
'loop_until': loop_until,
'parallel': parallel,
'name': name,
}
list_of_kwargs = list(callback_blitz_dispatcher_gen(**kwargs))
# increase the iterator_index until it hits the iterator_len
# it terminates the iteration when reaches the end of iterator
if iterator_len:
iterator_index += 1
# if parallel then create a pcall_payload of all the actions kwargs
# to call them later on in parallel
# NOTE: parallel would not work with until, do_until and loop_until
if parallel:
pcall_payload.extend(list_of_kwargs)
continue
ret_list.extend(list_of_kwargs)
# check if loop_until is true and get the item that meet the loop_until value
loop_until_last_index = _check_loop_until(self, ret_list, loop_until)
if loop_until_last_index is not None:
ret_list = ret_list[loop_until_last_index:loop_until_last_index +
1]
until_condition = _check_post_iteration(self, section, until, do_until,
loop_until_last_index)
_report_every_seconds(loop_start_time, every_seconds)
# execute each iteration of the loop in parallel
_actions_execute_in_loop_in_parallel(self, section, pcall_payload,
ret_list, steps)
return ret_list
def verify_cef_internal_label_stack(device, vrf, prefix, stack, max_time=60, check_interval=15):
""" Verify stack is programmed for prefix
Args:
device (`obj`): Device object
vrf (`str`): VRF to check
prefix (`str`): Prefix to check
stack (`list`): Stack list to verify exists
max_time (`int`): Maximum time to keep checking
check_interval (`int`): How long to wait between checks
Raises:
N/A
Returns:
True/False
"""
reqs = R(
[
"vrf",
"(?P<vrf>{vrf})".format(vrf=vrf),
"address_family",
"(?P<address_family>.*)",
"prefix",
"(?P<prefix>.*)",
"output_chain",
"tag_midchain",
"(?P<tag_midchain>.*)",
"label",
"(?P<label>.*)"
]
)
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = device.parse('show ip cef vrf {vrf} {prefix} internal'.format(vrf=vrf, prefix=prefix))
except SchemaEmptyParserError:
log.info("Parser output is empty")
timeout.sleep()
continue
stack_copy = list(stack)
found = find([out], reqs, filter_=False, all_keys=True)
if found:
for label in found[0][0]:
for stack_item in stack_copy:
if str(stack_item) in label:
# If item found, break to prevent else block from running
stack_copy.remove(stack_item)
break
else:
# If the label is not any of the stack_items break
# to prevent running next else block
break
else:
# If items exist in stack copy that means the stack
# from output had less items than expected
if not stack_copy:
return True
else:
log.info('The following labels are not in the output: {labels}'
.format(labels=stack_copy))
timeout.sleep()
return False
def _output_query_template(self,
output,
steps,
device,
command,
include,
exclude,
max_time,
check_interval,
continue_,
action,
reply=None,
arguments=None,
rest_device_alias=None,
rest_kwargs={}):
keys = _include_exclude_list(include, exclude)
max_time, check_interval = _get_timeout_from_ratios(
device=device, max_time=max_time, check_interval=check_interval)
timeout = Timeout(max_time, check_interval)
for query, style in keys:
# dict that would be sent with various data for inclusion/exclusion check
kwargs = {}
send_cmd = False
# for each query and style
with steps.start("Verify that '{query}' is {style} in the output".\
format(query=query, style=style), continue_=continue_) as substep:
while True:
if send_cmd:
output = _send_command(command,
device,
action,
arguments=arguments,
reply=reply,
rest_device_alias=rest_device_alias,
rest_kwargs=rest_kwargs)
if action == 'execute':
# validating the inclusion/exclusion of action execute,
pattern = re.compile(str(query))
found = pattern.search(str(output))
kwargs.update({
'action_output': found,
'operation': None,
'expected_value': None,
'style': style,
'key': query,
'query_type': 'execute_query'
})
else:
# verifying the inclusion/exclusion of actions : learn, parse and api
found = _get_output_from_query_validators(output, query)
kwargs = found
kwargs.update({'style': style, 'key': None})
# Function would return (pass | fail | error)
step_result, message = _verify_include_exclude(**kwargs)
if step_result == Passed:
substep.passed(message)
send_cmd = True
timeout.sleep()
if not timeout.iterate():
break
# failing logic in case of timeout
substep.failed(message)
return output
def verify_services_accounting_flow(device,
expected_flow_packets_ten_second_rate=None,
expected_active_flows=None,
max_time=60,
check_interval=10,
invert=False):
""" Verify 'show services accounting flow' against criteria
Args:
device (obj): Device object
expected_flow_packets_ten_second_rate (str): flow packets ten second rate
expected_active_flows (str): expected active flows
max_time (int, optional): Maximum timeout time. Defaults to 60.
check_interval (int, optional): Check interval. Defaults to 10.
invert (bool, optional): Inverts the API
Returns:
Boolean
Raises:
N/A
"""
op = operator.ne
if invert:
op = operator.eq
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
out = None
try:
out = device.parse('show services accounting flow')
except SchemaEmptyParserError:
timeout.sleep()
continue
#"flow-information": [
# {
# "interface-name": "ms-9/0/0",
# "local-ifd-index": "140",
# "flow-packets": "0",
# "flow-bytes": "0",
# "flow-packets-ten-second-rate": "0",
# "flow-bytes-ten-second-rate": "0",
# "active-flows": "0",
# "flows": "0",
# "flows-exported": "0",
# "flow-packets-exported": "9",
# "flows-expired": "0",
# "flows-aged": "0",
# }
#]
if expected_flow_packets_ten_second_rate and op(
out.q.get_values('flow-packets-ten-second-rate', 0),
str(expected_flow_packets_ten_second_rate)):
timeout.sleep()
continue
if expected_active_flows and op(out.q.get_values('active-flows', 0),
str(expected_active_flows)):
timeout.sleep()
continue
return True
return False
def verify_ip_and_sid_in_segment_routing_mapping_server(
device,
address_sid_dict,
address_family,
algorithm,
mapping_server,
max_time=300,
check_interval=30,
expected_result=True,
output=None):
""" Verifies if IP address and SID is present in Segment Routing mapping server
from show segment-routing mpls mapping-server {address_family}'
Args:
device (`obj`): Device to be executed command
address_family (`str`): Address family
address_sid_dict (`dict`): Dictionary containing ip address and SID as key and value pair
ex.)
{
'10.4.1.1/32': 1,
'10.4.1.2/32': 2,
}
algorithm (`str`): Algorithm to check
ex.)
algorithm = 'ALGO_0'
mapping_server (`str`): mapping server to check
ex.)
mapping_server = 'PREFIX_SID_EXPORT_MAP' or
mapping_server = 'PREFIX_SID_REMOTE_EXPORT_MAP'
max_time ('int'): maximum time to wait
check_interval ('int'): how often to check
expected_result ('bool'): Expected result
set expected_result = False if method should fail
set expected_result = True if method should pass (default value)
Raises:
None
Returns
True/False
"""
mapping_dict_export = {
'ipv4': 'ipv4_prefix_sid_export_map',
'ipv6': 'ipv6_prefix_sid_export_map',
}
mapping_dict_remote_export = {
'ipv4': 'ipv4_prefix_sid_remote_export_map',
'ipv6': 'ipv6_prefix_sid_remote_export_map',
}
timeout = Timeout(max_time, check_interval)
while timeout.iterate():
try:
out = None
if output:
out = device.parse(
"show segment-routing mpls mapping-server {}".format(
address_family),
output=output)
else:
out = device.parse(
"show segment-routing mpls mapping-server {}".format(
address_family))
output = None
except (SchemaEmptyParserError):
pass
found = None
for ip_address, sid in address_sid_dict.items():
# find using Prefix SID local
# It will use ipv4_prefix_sid_local or ipv6_prefix_sid_local as key for search data
# based on address_family provided
if out:
reqs = R([
'segment_routing', 'bindings', 'mapping_server', 'policy',
mapping_server.lower(), address_family, 'mapping_entry',
ip_address, 'algorithm', algorithm, 'sid', sid
])
found = find([out], reqs, filter_=False, all_keys=True)
# Returns false if SID is not found Prefix SID or Prefix SID local
if not expected_result and not found:
return expected_result
if expected_result and found:
return expected_result
if not found:
timeout.sleep()
return False