def get_clean_function(clean_name, clean_data, device):
"""From a clean function and device, return the function object"""
# Support calling multiple time the same section
name = clean_name.split('__')[0]
# For legacy reasons support calling stage by camelcase or snakecase.
# Example: ChangeBootVariable or change_boot_variable
if '_' in name or name == name.lower():
name = ''.join(word.title() for word in name.split('_'))
try:
data = clean_data[name]
except KeyError:
raise Exception(f"The clean stage '{name}' does not exist in the json "
f"file") from None
# Load abstraction tokens
tokens = Lookup.tokens_from_device(device)
# Start by checking the lowest level in the json using the abstraction tokens.
# For each consecutive iteration, remove the last token, checking every level
# until there is nothing left to check or a stage is found.
iterated_data = data
for i in reversed(range(1, len(tokens)+1)):
for token in tokens[:i]:
if token not in iterated_data:
break
iterated_data = iterated_data[token]
if 'module_name' in iterated_data:
# Found an abstracted stage
break
# reset for the next iteration
iterated_data = data
if iterated_data == data:
# The stage was not found under any of the abstraction tokens.
# Try 'com' as a last resort.
iterated_data = iterated_data.get('com', {})
if 'package' in iterated_data:
pkg = importlib.import_module(iterated_data['package'])
else:
pkg = clean
lookup = Lookup.from_device(device, packages={"clean": pkg})
try:
return getattr(_get_submodule(lookup.clean, iterated_data["module_name"]), name)
except Exception:
raise Exception(f"The clean stage '{name}' does not exist under the "
f"following abstraction tokens: {['com']+tokens}") from None
def get_parser(command, device):
'''From a show command and device, return parser class and kwargs if any'''
kwargs = {}
if command in parser_data:
# Then just return it
lookup = Lookup.from_device(device, packages={'parser': parser})
# Check if all the tokens exists; take the farthest one
data = parser_data[command]
for token in lookup._tokens:
if token in data:
data = data[token]
try:
return _find_parser_cls(device, data), kwargs
except KeyError:
# Case when the show command is only found under one of
# the child level tokens
raise Exception("Could not find parser for "
"'{c}' under {l}".format(
c=command, l=lookup._tokens)) from None
else:
# Regex world!
try:
found_data, kwargs = _find_command(command, parser_data, device)
except SyntaxError:
# Could not find a match
raise Exception("Could not find parser for "
"'{c}'".format(c=command)) from None
return _find_parser_cls(device, found_data), kwargs
def learn(feature, dev):
if not dev.is_connected():
dev.connect()
abstract = Lookup.from_device(dev)
# Find the Interface Parsers for this device
# 1) Directory must exists in genie.libs.ops.<feature>
# 2) Then abstraction will kick in to find the right one
# 3) The directory syntax is <feature>.<feature.<Feature>
# Where the class is capitalized but the directory/files arent.
# e.g interface.interface.Interface
class_name = '.'.join(
['abstract.ops',
feature.lower(),
feature.lower(),
feature.title()])
try:
feature_details = eval(class_name)(dev)
except LookupError:
raise NotImplementedError("Feature is not implemented for this OS")
feature_details.learn()
if hasattr(feature_details, 'info'):
return feature_details.info
else:
return feature_details
def learn_system_defaults(self, testbed):
"""Execute commands to learn default system information
Args:
testbed (`obj`): Testbed object
Returns:
None
Raises:
pyATS Results
"""
# Get default memory location
self.parent.default_file_system = {}
for device in self.parent.mapping_data['devices']:
dev = testbed.devices[device]
lookup = Lookup.from_device(dev)
try:
self.parent.default_file_system[dev.name] = lookup.sdk.libs.\
abstracted_libs.subsection.get_default_dir(device=dev)
except LookupError as e:
log.info('Cannot find device {d} correspoding get_default_dir'.format(d=dev.name))
except Exception as e:
self.failed('Unable to learn system defaults', from_exception=e)
if not self.parent.default_file_system:
self.failed('Unable to set default directory')
def connect(self, testbed, testscript, steps, context, pool_num):
'''Connect to the devices, using either Cli and/or Yang'''
# Connect to all the devices
with steps.start('Connect to all devices'):
for dev in testbed.devices.values():
# If the context of this device is Yang, then connect via Yang
if dev.context == 'yang':
time.sleep(5)
dev.connect(alias='nc', via='netconf')
# As the alias name can be anything, We need to tell the
# infrastructure what is the alias of the yang connection
dev.mapping['yang'] = 'nc'
time.sleep(5)
# Cli supports a pool device mechanism. This will allow Ops
# to learn the features faster.
if pool_num > 1:
dev.start_pool(alias='vty', via='cli', size=pool_num)
else:
dev.connect(via='a', alias='cli')
# As the alias name can be anything, We need to tell the
# infrastructure what is the alias of the cli connection
# Use connection a for cli communication to the device
dev.mapping['cli'] = 'a'
# Abstraction
# Right now abstraction is done via OS and Context.
dev.lib = Lookup(dev.os,
dev.context,
packages={
'conf': conf,
'ops': ops
})
def configure_replace(self, testbed, steps, devices, timeout=60):
for name, dev in devices.items():
try:
device = testbed.devices.get(name, None)
if not device or not device.is_connected():
continue
file_name = None
file_location = None
lookup = Lookup.from_device(device)
if 'file_location' in dev:
file_location = dev['file_location']
else:
file_location = lookup.sdk.libs.\
abstracted_libs.subsection.get_default_dir(
device=device)
if 'file_name' not in dev:
log.error('Missing file_name for device {}'.format(name))
continue
if 'file_name' in dev:
file_name = dev['file_name']
lookup.sdk.libs.abstracted_libs.subsection.configure_replace(
device,
file_location,
timeout=dev.get('timeout', timeout),
file_name=file_name)
except Exception as e:
self.failed("Failed to replace config : {}".format(str(e)))
log.info("Configure replace is done for device {}".format(name))
def asynchronous_save_boot_variable(self, device, device_dict):
'''Use asynchronous execution when saving boot variables on devices'''
log.info(
banner("Check boot information to see if they are consistent\n"
"and save bootvar to startup-config on device '{d}'".format(
d=device.name)))
# get platform pts
platform_pts = self.parameters.get('pts',
{}).get('platform',
{}).get(device.name, None)
try:
result = Lookup.from_device(device).sdk.libs.abstracted_libs.subsection.\
save_device_information(device=device, platform_pts=platform_pts)
except Exception as e:
device_dict[device.name] = 'Failed'
else:
if result == 'Skipped':
device_dict[device.name] = 'Skipped'
else:
device_dict[device.name] = 'Passed'
return device_dict
def exclude_management_interface(self, device, requirements, ops_obj):
'''Exclude the management interface from the trigger interfaces'''
# exclude the managemnet interface from the interfaces
# learned for the trigger action. Unless specified
# by the user to include the management interafce
# "user specifies include_management_interface as True"
if 'include_management_interface' in requirements and \
requirements['include_management_interface']:
# management interface will be considered in the
# trigger action
return ops_obj
else:
Interface_ops_obj = Lookup.from_device(device).ops.interface.interface.Interface(device)
new_dict = {}
if isinstance(ops_obj, type(Interface_ops_obj)) and \
device.management_interface:
new_dict['info'] = {}
for key in ops_obj.info.keys():
if device.management_interface != key:
new_dict['info'][key] = ops_obj.info[key]
# management interface will be excluded from the trigger action
if new_dict:
return new_dict
else:
return ops_obj
def save_bootvar(self, testbed):
"""Check boot information and save bootvar to startup-config
Args:
testbed (`obj`): Testbed object
Returns:
None
Raises:
pyATS Results
"""
log.info(banner('Check boot information to see if they are consistent\n'
'and save bootvar to startup-config'))
# get uut
devices = testbed.find_devices(alias='uut')
for uut in devices:
lookup = Lookup.from_device(uut)
# get platform pts
platform_pts = self.parameters.get('pts', {}).get('platform', {}).get('uut', None)
try:
lookup.sdk.libs.abstracted_libs.subsection.save_device_information(
device=uut, platform_pts=platform_pts)
except Exception as e:
self.passx('Failed to save boot var or copy running-config to startup-config',
from_exception=e)
def _get_clean(clean_name, clean_data, device):
"""From a clean function and device, return the function object"""
# Support calling multiple time the same section
name = clean_name.split('__')[0]
try:
data = clean_data[name]
except KeyError:
raise Exception("Could not find a clean stage called '{c}'".format(
c=name)) from None
# Load SDK abstraction
lookup = Lookup.from_device(device, packages={"clean": clean})
# if this is true after below loop, it means the function not under any os
is_com = True
# find the token in the lowest level of the json
for token in lookup._tokens:
if token in data:
data = data[token]
is_com = False
# if not found, search under 'com' token
if is_com:
data = data['com']
try:
mod = getattr(_get_submodule(lookup.clean, data["module_name"]), name)
mod.__wrapped__.__name__ = clean_name
return mod
except Exception:
raise Exception("Could not find '{cn}' clean section under '{o}', "
"and common".format(cn=name, o=device.os)) from None
def get_default_dir(device):
""" Get the default directory of this device
Args:
Mandatory:
device (`obj`) : Device object.
Returns:
default_dir (`str`): Default directory of the system
Raises:
Exception
Example:
>>> get_default_dir(device=device)
"""
try:
lookup = Lookup.from_device(device)
parsed_dict = lookup.parser.show_platform.Dir(device=device).parse()
default_dir = parsed_dict['dir']['dir'].replace('/', '')
except SchemaEmptyParserError as e:
raise Exception("No output when executing 'dir' command") from e
except Exception as e:
raise Exception("Unable to execute or parse 'dir' command") from e
# Return default_dir to caller
log.info("Default directory on '{d}' is '{dir}'".format(d=device.name,
dir=default_dir))
return default_dir
def _find_command(command, data, device):
for key in data:
if not '{' in key:
# Disregard the non regex ones
continue
# Okay... this is not optimal
patterns = re.findall('{.*?}', key)
reg = key
for pattern in patterns:
word = pattern.replace('{', '').replace('}', '')
new_pattern = '(?P<{p}>.*)'.format(p=word)
reg = re.sub(pattern, new_pattern, key)
match = re.match(reg, command)
if match:
# Found a match!
lookup = Lookup.from_device(device, packages={'parser': parser})
# Check if all the tokens exists; take the farthest one
ret_data = data[key]
for token in lookup._tokens:
if token in ret_data:
ret_data = ret_data[token]
return (ret_data, match.groupdict())
raise SyntaxError('Could not find a parser match')
def __init__(self, device=None):
self.abstract = Lookup.from_device(device,
packages={
'sdk': sdk,
'parser': parser
})
self.lib = self.abstract.sdk.libs.abstracted_libs.restore.Restore()
def _configure_replace_util(device, dev, device_name):
try:
file_name = None
file_location = None
lookup = Lookup.from_device(device)
#overriding the default directory
if 'file_location' in dev:
file_location = dev['file_location']
log.warning("Overriding the default directory with {}".format(
file_location))
else:
file_location = lookup.sdk.libs. \
abstracted_libs.subsection.get_default_dir(
device=device)
if 'file_name' not in dev:
log.error(
'Missing file_name for device {}'.format(device_name))
return
if file_location and file_location[-1:] != '/':
file_location = file_location + '/'
if 'file_name' in dev:
file_name = dev['file_name']
lookup.sdk.libs.abstracted_libs.subsection.configure_replace(
device,
file_location,
timeout=dev.get('timeout', timeout),
file_name=file_name)
except Exception as e:
self.failed("Failed to replace config : {}".format(str(e)))
log.info("Configure replace is done for device {}".format(device_name))
def get_image_handler(device):
if device.clean.get('images'):
# Get abstracted ImageHandler class
abstract = Lookup.from_device(device, packages={'clean': clean})
ImageHandler = abstract.clean.stages.image_handler.ImageHandler
return ImageHandler(device, device.clean['images'])
else:
return None
def learn_the_system(self, testbed, steps, features=None):
"""Learn and store the system properties
Args:
testbed (`obj`): Testbed object
steps (`obj`): aetest steps object
features (`dict`): dict of components and the feature that contains the component.
ex. {'pim': ['autorp',],
'bgp': ['confederationpeers', 'gracefulrestart']}
Returns:
None
Raises:
pyATS Results
"""
log.info(
banner('Learn and store platform information, lldp neighbors'
', from PTS if PTS is existed, otherwise from show commands'))
# get uut, having a uut is mandatory in Genie
uut = testbed.devices['uut']
lookup = Lookup.from_device(uut)
# get platform PTS
platform_pts = self.parameters.get('pts', {}).get('platform',
{}).get('uut', None)
with steps.start(
"Store and learn platform information from 'show lldp neighbors detail' on {}"
.format(self.name)) as step:
try:
lookup.sdk.libs.abstracted_libs\
.subsection.learn_system(device=uut, steps=steps, platform_pts=platform_pts)
except Exception as e:
step.passx('Cannot Learn and Store system info', from_exception=e)
# learn platform lldp neighbors
with steps.start("learn platform lldp neighbors on device {}".format(
uut.name)) as step:
# inital lldp ops object
lldp_ops = lookup.ops.lldp.lldp.Lldp(
uut,
attributes=['info[interfaces][(.*)][neighbors][(.*)][port_id]'])
# learn the lldp ops
try:
lldp_ops.learn()
except Exception as e:
step.passx('Cannot learn lldp information', from_exception=e)
if not hasattr(lldp_ops, 'info'):
step.passx('No LLDP neighbors')
# store the lldp information
uut.lldp_mapping = lldp_ops.info['interfaces']
def check_issu_state(cls,
device,
slot,
expected_state,
attempt=3,
sleep=5):
''' Check if the ISSU state is in the expected state
Args:
device (`obj`): Device Object.
expected_state (`str`): Acceptable ISSU states are:
- loadversion
- runversion
- acceptversion
- commitversion
slot (`str`): Slot for which we need to check ISSU state
attempt (`int`): Attempt numbers when learn the feature.
sleep (`int`): The sleep time.
Returns:
None
Raises:
AssertionError: 'expected_state' is not as expected
Exception: Cannot parse 'show issu state detail' output
No output form 'show issu state detail'
Unable to execute 'show issu state detail'
Example:
>>> check_issu_state(device=uut, slot='R1',
expected_state='commitversion')
'''
assert expected_state in [
'loadversion', 'runversion', 'acceptversion', 'commitversion'
]
lookup = Lookup.from_device(device)
for i in range(attempt):
try:
issu_dict = lookup.parser.show_issu.\
ShowIssuStateDetail(device=device).parse()
rs = R(['slot', slot, 'last_operation', expected_state])
ret = find([issu_dict], rs, filter_=False, all_keys=True)
if ret:
break
except SchemaEmptyParserError as e:
raise Exception(
"No output or unable to parse 'show issu state "
"detail'",
from_exception=e)
except Exception as e:
raise Exception("Unable to execute 'show issu state detail'",
from_exception=e)
time.sleep(sleep)
else:
raise AssertionError("FAIL: ISSU state not '{}' - this is "
"unexpected".format(expected_state))
def __init__(self, *args, **kwargs):
'''__init__ instantiates a single connection instance.'''
super().__init__(*args, **kwargs)
# Set up abstraction for this device
lookup = Lookup.from_device(self.device, default_tokens=['os'])
_implementation = lookup.libs.implementation.Implementation
self._implementation = _implementation(*args, **kwargs)
def get_parser(command, device, fuzzy=False):
'''From a show command and device, return parser class and kwargs if any'''
try:
order_list = device.custom.get('abstraction').get('order', [])
except AttributeError:
order_list = None
lookup = Lookup.from_device(device, packages={'parser': parser})
results = _fuzzy_search_command(command, fuzzy, device.os, order_list)
valid_results = []
for result in results:
found_command, data, kwargs = result
if found_command == 'tokens':
continue
# Check if all the tokens exists and take the farthest one
for token in lookup._tokens:
if token in data:
data = data[token]
try:
valid_results.append(
(found_command, _find_parser_cls(device, data), kwargs))
except KeyError:
# Case when the show command is only found under one of
# the child level tokens
continue
if not valid_results:
'''result is not valid. raise custom ParserNotFound exception'''
raise ParserNotFound(command, lookup._tokens)
# Try to add parser to telemetry data
if INTERNAL:
try:
# valid_results is a list of found parsers for a given show command
# - first element in this list is the closest parser match found
# - each element has the format (show command, class, kwargs)
# valid_results[0] is the best parser match
add_parser_usage_data(valid_results[0], device)
except Exception as e:
log.debug("Encountered an unexpected error while adding parser "
"telemetry data: %s" % e)
if not fuzzy:
# valid_results is a list of found parsers for a given show command
# - first element in this list is the closest parser match found
# - each element has the format (show command, class, kwargs)
# valid_results[0][1] is the class of the best match
# valid_results[0][2] is a dict of parser kwargs
return valid_results[0][1], valid_results[0][2]
return valid_results
def _find_command(command, data, device):
ratio = 0
max_lenght = 0
matches = None
for key in data:
if not '{' in key:
# Disregard the non regex ones
continue
# Okay... this is not optimal
patterns = re.findall('{.*?}', key)
len_normal_words = len(set(key.split()) - set(patterns))
reg = key
for pattern in patterns:
word = pattern.replace('{', '').replace('}', '')
new_pattern = r'(?P<{p}>\\S+)'.format(p=word) \
if word == 'vrf' \
or word == 'rd' \
or word == 'instance' \
or word == 'vrf_type' \
or word == 'feature' \
else '(?P<{p}>.*)'.format(p=word)
reg = re.sub(pattern, new_pattern, reg)
reg += '$'
# Convert | to \|, and ^ to \^
reg = reg.replace('|', '\|').replace('^', '\^')
match = re.match(reg, command)
if match:
try:
order_list = device.custom.get('abstraction').get('order', [])
except AttributeError:
order_list = None
if order_list:
if getattr(device, order_list[0]) not in data[key].keys():
continue
elif device.os not in data[key].keys():
continue
# Found a match!
lookup = Lookup.from_device(device, packages={'parser':parser})
# Check if all the tokens exists; take the farthest one
ret_data = data[key]
for token in lookup._tokens:
if token in ret_data:
ret_data = ret_data[token]
if len_normal_words > max_lenght:
max_lenght = len_normal_words
matches = (ret_data, match.groupdict())
if matches:
return matches
raise SyntaxError('Could not find a parser match')
def learn_interfaces(self):
""" Sample test section. Only print """
self.all_interfaces = {}
for dev in self.parent.parameters['dev']:
log.info(banner("Gathering Interface Information from {}".format(
dev.name)))
abstract = Lookup.from_device(dev)
intf = abstract.ops.interface.interface.Interface(dev)
intf.learn()
self.all_interfaces[dev.name] = intf.info
def learn_bgp(self):
""" Sample test section. Only print """
self.all_bgp_sessions = {}
for dev in self.parent.parameters['dev']:
log.info(
banner("Gathering BGP Information from {}".format(dev.name)))
abstract = Lookup.from_device(dev)
bgp = abstract.ops.bgp.bgp.Bgp(dev)
bgp.learn()
self.all_bgp_sessions[dev.name] = bgp.info
def get_routing_table(dev):
"""
returns a parsed and normalized routing table
"""
if not dev.is_connected():
dev.connect()
abstract = Lookup.from_device(dev)
# The directory syntax is <feature>.<feature.<Feature>
routing = abstract.ops.routing.routing.Routing(dev)
routing.learn()
return routing.info
def learn_ospf(self):
""" For each device in testbex, learn OSPF state. OSPF details used in later tests. """
self.all_ospf_sessions = {}
for dev in self.parent.parameters["dev"]:
log.info(
banner("Gathering OSPF Information from {}".format(dev.name)))
abstract = Lookup.from_device(dev)
ospf = abstract.ops.ospf.ospf.Ospf(dev)
ospf.learn()
self.all_ospf_sessions[dev.name] = ospf
def genie_prep(dev):
"""
Connects and looks up platform parsers for device
Returns an abstract object
"""
# Device must be connected so that a lookup can be performed
if not dev.is_connected():
dev.connect()
# Load the approprate platform parsers dynamically
abstract = Lookup.from_device(dev)
return abstract
def check_issu_rollback_timer(cls,
device,
slot,
expected_state,
attempt=3,
sleep=5):
''' Check if the ISSU state is in the expected state
Args:
device (`obj`): Device Object.
expected_state (`str`): Acceptable ISSU states are:
- active
- inactive
slot (`str`): Slot for which we need to check ISSU rollback timer
attempt (`int`): Attempt numbers when learn the feature.
sleep (`int`): The sleep time.
Returns:
None
Raises:
AssertionError: 'expected_state' is not as expected
Exception: Cannot parse 'show issu rollback timer' output
No output form 'show issu rollback timer'
Unable to execute 'show issu rollback timer'
Example:
>>> check_issu_rollback_timer(device=uut, slot='R1',
expected_state='inactive')
'''
assert expected_state in ['active', 'inactive']
lookup = Lookup.from_device(device)
for i in range(attempt):
try:
issu_dict = lookup.parser.show_issu.\
ShowIssuRollbackTimer(device=device).parse()
if issu_dict and 'rollback_timer_state' in issu_dict and\
issu_dict['rollback_timer_state'] == expected_state:
break
except SchemaEmptyParserError as e:
raise Exception(
"No output or unable to parse 'show issu "
"rollback timer'",
from_exception=e)
except Exception as e:
raise Exception("Unable to execute 'show issu rollback timer'",
from_exception=e)
time.sleep(sleep)
else:
raise AssertionError("FAIL: ISSU rollback timer not '{}' - "
"this is unexpected".format(expected_state))
def learn_the_system(self, testbed, steps):
"""Learn and store the system properties
Args:
testbed (`obj`): Testbed object
steps (`obj`): aetest steps object
Returns:
None
Raises:
pyATS Results
"""
log.info(banner('Learn and store platform information, lldp neighbors'
', from PTS if PTS is existed, otherwise from show commands'))
# get uut
devices = testbed.find_devices(alias='uut')
for uut in devices:
lookup = Lookup.from_device(uut)
# get platform PTS
platform_pts = self.parameters.get('pts', {}).get('platform', {}).get('uut', None)
try:
lookup.sdk.libs.abstracted_libs\
.subsection.learn_system(device=uut, steps=steps, platform_pts=platform_pts)
except Exception as e:
step.passx('Cannot Learn and Store system info',
from_exception=e)
# learn platform lldp neighbors
with steps.start("learn platform lldp neighbors on device {}"
.format(uut.name)) as step:
# inital lldp ops object
lldp_ops = lookup.ops.lldp.lldp.Lldp(
uut, attributes=['info[interfaces][(.*)][neighbors][(.*)][port_id]'])
# learn the lldp ops
try:
lldp_ops.learn()
except Exception as e:
step.passx('Cannot learn lldp information',
from_exception=e)
if not hasattr(lldp_ops, 'info'):
step.passx('No LLDP neighbors')
# store the lldp information
uut.lldp_mapping = lldp_ops.info['interfaces']
def configure_ospf(device, router_id_digit, unconfig=False):
lookup = Lookup.from_device(device, packages={'conf': conf, 'ops': ops})
ospf = conf.ospf.ospf.Ospf()
ospf.device_attr[device].vrf_attr['default'].instance = '9'
ospf.device_attr[device].vrf_attr['default'].router_id = '9.9.9.{}'.format(\
router_id_digit)
device.add_feature(ospf)
if unconfig:
print('Removing configuration for {d}'.format(d=device))
ospf.build_unconfig()
else:
print('Applying configuration for {d}'.format(d=device))
ospf.build_config()
def retrieve_ospf(device):
lookup = Lookup.from_device(device)
ospf = lookup.ops.ospf.ospf.Ospf(device)
ospf.learn()
# Compare of take a snapshot
if hasattr(device, 'snapshot'):
diff = ospf.diff(device.snapshot,
exclude=['age', 'dead_timer', 'hello_timer'])
if diff:
print('Found some difference for device {d}'.format(d=device))
print(diff)
else:
device.snapshot = ospf
def __new__(cls, device, *args, **kwargs):
if '.'.join([cls.__module__, cls.__name__]) == \
'genie.trafficgen.trafficgen.TrafficGen':
if hasattr(device, 'platform') and device.platform:
tgen_abstract = Lookup.from_device(
device,
packages={'tgn': trafficgen},
default_tokens=['os', 'platform'])
try:
new_cls = getattr(tgen_abstract.tgn,
device.platform).TrafficGen
except LookupError:
new_cls = tgen_abstract.tgn.TrafficGen
else:
tgen_abstract = Lookup.from_device(
device,
packages={'tgn': trafficgen},
default_tokens=['os'])
new_cls = tgen_abstract.tgn.TrafficGen
return super().__new__(new_cls)
else:
return super().__new__(cls)
