def _load_parser_json():
'''get all parser data in json file'''
try:
mod = importlib.import_module('genie.libs.parser')
parsers = os.path.join(mod.__path__[0], 'parsers.json')
except Exception:
parsers = ''
if not os.path.isfile(parsers):
log.warning('parsers.json does not exist, make sure you '
'are running with latest version of '
'genie.libs.parsers')
parser_data = {}
else:
# Open all the parsers in json file
with open(parsers) as f:
parser_data = json.load(f)
# check if provided external parser packages
ext_parser_package = cfg.get(PYATS_EXT_PARSER, None) or \
os.environ.get(PYATS_EXT_PARSER.upper().replace('.', '_'))
if ext_parser_package:
ext = ExtendParsers(ext_parser_package)
ext.extend()
ext.output.pop('tokens', None)
summary = ext.output.pop('extend_info', None)
merge_dict(parser_data, ext.output, update=True)
log.warning("External parser counts: {}\nSummary:\n{}".format(
len(summary), json.dumps(summary, indent=2)))
return parser_data
def get_policy_map_configurational_policy_map(device, interfaces):
""" Get policy-map running configuration
Args:
device (`obj`): Device object
interfaces (`list`): List of interfaces
Returns:
policy-map configurational dictionary
"""
out = {}
policy_out = {}
for interface in interfaces:
out[interface] = get_interface_running_config(device, interface)
service_policies = []
for item in out[interface]:
if interface in item:
for service_policy_item in out[interface][item]:
if "service-policy" in service_policy_item:
service_policies.append(service_policy_item[21:].strip())
for service_policy in service_policies:
if "in" in service_policy:
direction = "input"
else:
direction = "output"
output = get_policy_map_running_policy_map(device, service_policy)
if not output:
continue
policy_out.setdefault(direction, {})
policy_out[direction] = output
for class_name in policy_out[direction]["policy_map"][service_policy][
"class"]:
for item in policy_out[direction]["policy_map"][service_policy][
"class"][class_name]:
if "service_policy" in item:
nested = policy_out[direction]["policy_map"][
service_policy]["class"][class_name][item]
nested_policy_out = get_policy_map_running_policy_map(
device, nested)
if not nested_policy_out:
continue
new_nested_policy_out = {}
new_nested_policy_out.setdefault(
"policy_map",
{}).setdefault(service_policy,
{}).setdefault("child_policy_name", {})
new_nested_policy_out["policy_map"][service_policy][
"child_policy_name"] = nested_policy_out["policy_map"]
merge_dict(policy_out[direction], new_nested_policy_out)
return policy_out
def load_clean_json():
"""get all clean data in json file"""
try:
mod = importlib.import_module("genie.libs.clean")
functions = os.path.join(mod.__path__[0], "clean.json")
except Exception:
functions = ""
if not os.path.isfile(functions):
log.warning(
"clean.json does not exist, make sure you "
"are running with latest version of "
"genie.libs.clean"
)
clean_data = {}
else:
# Open all the parsers in json file
with open(functions) as f:
clean_data = json.load(f)
for entry in iter_entry_points(group=CLEAN_PLUGIN_ENTRYPOINT):
log.info('Loading clean APIs from {}'.format(entry.module_name))
ext = ExtendClean(entry.module_name)
ext.extend()
ext.output.pop('tokens', None)
log.info("{} clean API count: {}".format(
entry.module_name,
len(ext.output.keys())))
log.debug('{} clean APIs {}'.format(
entry.module_name,
json.dumps(ext.output, indent=4)
))
plugin_clean_data = ext.output
clean_data = merge_dict(clean_data, plugin_clean_data, update=True)
return clean_data
def yang_cli(self):
cli_output = self.cli()
yang_output = self.yang()
merged_output = merge_dict(yang_output, cli_output)
return merged_output
def learn(self):
'''Learn Platform object'''
self.add_leaf(cmd=show_platform.ShowInventory,
src='[name][(?P<slot>.*)][description]',
dest='Slot[(?P<slot>.*)][description]')
self.add_leaf(cmd=show_platform.ShowInventory,
src='[name][(?P<slot>.*)][serial_number]',
dest='Slot[(?P<slot>.*)][serial_number]')
self.add_leaf(cmd=show_platform.ShowInventory,
src='[name][(?P<slot>.*)][pid]',
dest='Slot[(?P<slot>.*)][pid]')
self.add_leaf(cmd=show_platform.ShowInventory,
src='[name][(?P<slot>.*)][slot]',
dest='Slot[(?P<slot>.*)][slot]')
# Place holder to make it more readable
src_vdc_detail = '[vdc][(?P<id>.*)]'
dest_vdc_detail = '[virtual_device][(?P<id>.*)]'
self.add_leaf(cmd=show_platform.ShowVdcDetail,
src=src_vdc_detail + '[name]',
dest=dest_vdc_detail + '[vd_name]')
self.add_leaf(cmd=show_platform.ShowVdcDetail,
src=src_vdc_detail + '[state]',
dest=dest_vdc_detail + '[vd_status]')
# Place holder to make it more readable
src_vdc_membership = '[virtual_device][(?P<id>.*)][membership]'
dest_vdc_membership = '[virtual_device][(?P<id>.*)][membership]'
self.add_leaf(cmd=show_platform.ShowVdcMembershipStatus,
src=src_vdc_membership +
'[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][vd_ms_status]',
dest=dest_vdc_membership +
'[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][status]')
self.add_leaf(cmd=show_platform.ShowVdcMembershipStatus,
src=src_vdc_membership +
'[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][vd_ms_type]',
dest=dest_vdc_membership +
'[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][type]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[xbar]',
dest='[slot][oc]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[xbar][(?P<xbar>.*)][status]',
dest='[slot][oc][(?P<xbar>.*)][state]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[xbar][(?P<xbar>.*)][module_type]',
dest='[slot][oc][(?P<xbar>.*)][name]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[xbar]',
dest='[xbar]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[slot]',
dest='[module]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][model]',
dest='[slot][rp][(?P<slot>.*)][name]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][status]',
dest='[slot][rp][(?P<slot>.*)][state]')
self.add_leaf(
cmd=show_platform.ShowModule,
src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][serial_number]',
dest='[slot][rp][(?P<slot>.*)][sn]')
self.add_leaf(cmd=show_platform.ShowModule,
src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][status]',
dest='[slot][rp][(?P<slot>.*)][redundancy_state]')
# 'subslot' is not available on NXOS
self.add_leaf(cmd=show_platform.Dir, src='[dir]', dest='[dir]')
self.add_leaf(cmd=show_platform.Dir,
src='[disk_used_space]',
dest='[disk_used_space]')
self.add_leaf(cmd=show_platform.Dir,
src='[disk_free_space]',
dest='[disk_free_space]')
self.add_leaf(cmd=show_platform.Dir,
src='[disk_total_space]',
dest='[disk_total_space]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][hardware][memory]',
dest='[main_mem]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][software][kickstart]',
dest='[kickstart_version]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][software][kickstart_image_file]',
dest='[kickstart_image]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][software][system_version]',
dest='[version]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][software][system_image_file]',
dest='[image]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][os]',
dest='[os]')
self.add_leaf(cmd=show_platform.ShowVersion,
src='[platform][hardware][model]',
dest='[rtr_type]')
self.add_leaf(
cmd=show_platform.ShowInstallActive,
src=
'[active_packages][active_package_module_0][active_package_name]',
dest='[installed_packages]')
self.add_leaf(cmd=show_platform.ShowRedundancyStatus,
src='[active_supervisor_time]',
dest='[rp_uptime]',
action=self.convert_to_seconds)
self.add_leaf(
cmd=show_platform.ShowBoot,
src=
'[next_reload_boot_variable][sup_number][sup-1][kickstart_variable]',
dest='[rp_kickstart_boot_image]')
self.add_leaf(
cmd=show_platform.ShowBoot,
src=
'[next_reload_boot_variable][sup_number][sup-1][system_variable]',
dest='[rp_boot_image]')
self.add_leaf(cmd=show_platform.ShowBoot,
src='[next_reload_boot_variable][kickstart_variable]',
dest='[rp_kickstart_boot_image]')
self.add_leaf(cmd=show_platform.ShowBoot,
src='[next_reload_boot_variable][system_variable]',
dest='[rp_boot_image]')
self.make(final_call=True)
if 'Slot' in self.__dict__:
line_cards = []
slot_value = None
new_dict = {}
new_dict['slot'] = {}
new_dict['chassis'] = self.Slot['Chassis']['description'].strip()
new_dict['chassis_sn'] = self.Slot['Chassis'][
'serial_number'].strip()
for slot in self.Slot:
if 'Slot' in slot:
slot_value = self.Slot[slot]['slot']
if 'Nexus' in self.Slot[slot]['description']:
slot_dict = new_dict.setdefault('slot', {}).setdefault(
'oc', {}).setdefault(slot_value, {})
slot_dict.update(
{'name': self.Slot[slot]['description']})
if 'serial_number' in self.Slot[slot]:
slot_dict.update(
{'sn': self.Slot[slot]['serial_number']})
elif 'Supervisor' not in self.Slot[slot]['description']:
slot_dict = new_dict.setdefault('slot', {}).setdefault(
'lc', {}).setdefault(slot_value, {})
slot_dict.update(
{'name': self.Slot[slot]['description']})
if 'serial_number' in self.Slot[slot]:
slot_dict.update(
{'sn': self.Slot[slot]['serial_number']})
elif 'Module' in slot:
slot_value = self.Slot[slot]['slot']
if 'Nexus' in self.Slot[slot]['description']:
slot_dict = new_dict.setdefault('slot', {}).setdefault(
'oc', {}).setdefault(slot_value, {})
slot_dict.update(
{'name': self.Slot[slot]['description']})
slot_dict.update(
{'sn': self.Slot[slot]['serial_number']})
if 'Supervisor' not in self.Slot[slot]['description']:
slot_dict = new_dict.setdefault('slot', {}).setdefault(
'lc', {}).setdefault(slot_value, {})
slot_dict.update(
{'name': self.Slot[slot]['description']})
slot_dict.update(
{'sn': self.Slot[slot]['serial_number']})
merge_dict(self.__dict__, new_dict)
del self.Slot
for slot_number in self.slot['rp']:
try:
self.slot['rp'][str(
slot_number)]['rp_boot_image'] = self.rp_boot_image
except Exception:
pass
try:
self.slot['rp'][str(
slot_number
)]['rp_kickstart_boot_image'] = self.rp_kickstart_boot_image
except Exception:
pass
self.slot['rp'][str(slot_number)]['rp_uptime'] = self.rp_uptime
if 'lc' in self.slot:
for item in self.slot['lc']:
if 'sn' in self.slot['lc'][item]:
serial = self.slot['lc'][item]['sn']
else:
serial = None
if 'module' in self.__dict__:
for key in self.module['lc']:
for mod_name in self.module['lc'][key]:
if 'serial_number' not in self.module['lc'][
key][mod_name]:
continue
if self.module['lc'][key][mod_name][
'serial_number'] == serial:
linecard_status = self.module['lc'][key][
mod_name]['status']
self.slot['lc'][item].update(
{'state': linecard_status})
if 'sup' not in self.slot['lc'][item]['name'].lower() and\
'ethernet' not in self.slot['lc'][item]['name'].lower():
self.slot.setdefault(
'oc',
{}).setdefault(item, {}).update(
self.slot['lc'][item])
self.slot['oc'][item].update(
{'state': linecard_status})
if 'xbar' in self.__dict__:
for key in self.xbar:
if 'serial_number' not in self.xbar[key]:
continue
if self.xbar[key]['serial_number'] == serial:
linecard_status = self.xbar[key]['status']
self.slot.setdefault('oc', {}).setdefault(
item,
{}).update({'state': linecard_status})
if 'oc' in self.slot:
for item in self.slot['oc']:
if 'sn' in self.slot['oc'][item]:
serial = self.slot['oc'][item]['sn']
else:
serial = None
if 'module' in self.__dict__:
for key in self.module['lc']:
for mod_name in self.module['lc'][key]:
if 'serial_number' not in self.module['lc'][
key][mod_name]:
continue
if self.module['lc'][key][mod_name][
'serial_number'] == serial:
linecard_status = self.module['lc'][key][
mod_name]['status']
self.slot['oc'][item].update(
{'state': linecard_status})
if 'xbar' in self.__dict__:
for key in self.xbar:
if 'serial_number' not in self.xbar[key]:
continue
if self.xbar[key]['serial_number'] == serial:
linecard_status = self.xbar[key]['status']
self.slot['oc'][item].update(
{'state': linecard_status})
if 'oc' in self.slot and 'lc' in self.slot:
if item in self.slot['oc'] and item in self.slot['lc']:
if self.slot['oc'][item] == self.slot['lc'][item]:
self.slot['lc'].pop(item)
try:
del self.rp_boot_image
except Exception:
pass
try:
del self.rp_kickstart_boot_image
except Exception:
pass
del self.module
try:
del self.xbar
except Exception:
pass
if 'virtual_device' in self.__dict__:
for virt_dev in self.virtual_device:
if 'membership' in self.virtual_device[virt_dev]:
for item in self.virtual_device[virt_dev]['membership']:
self.virtual_device[virt_dev]['membership'] = \
self.virtual_device[virt_dev]['membership'][item]
def create_yaml_files(self, datafile):
# Load existing Yaml file (If any)
# Load main verifications datafiles
#main_file = OrderedDict()
main_file = {}
nxos_file = {
'extends':
'%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml'
}
iosxe_file = {
'extends':
'%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml'
}
ios_file = {
'extends':
'%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml'
}
iosxr_file = {
'extends':
'%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml'
}
junos_file = {
'extends':
'%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml'
}
nxos = []
iosxe = []
ios = []
iosxr = []
junos = []
ios = []
# Load the file
with open(datafile, 'r') as f:
parser_yaml = yaml.safe_load(f)
main_yaml = os.path.join(YAMLS, 'verification_datafile.yaml')
with open(main_yaml, 'r') as f:
content = yaml.safe_load(f)
nxos_yaml = os.path.join(YAMLS, 'nxos',
'verification_datafile_nxos.yaml')
with open(nxos_yaml, 'r') as f:
nxos_content = yaml.safe_load(f)
iosxe_yaml = os.path.join(YAMLS, 'iosxe',
'verification_datafile_iosxe.yaml')
with open(iosxe_yaml, 'r') as f:
iosxe_content = yaml.safe_load(f)
ios_yaml = os.path.join(YAMLS, 'ios', 'verification_datafile_ios.yaml')
with open(ios_yaml, 'r') as f:
ios_content = yaml.safe_load(f)
iosxr_yaml = os.path.join(YAMLS, 'iosxr',
'verification_datafile_xr.yaml')
with open(iosxr_yaml, 'r') as f:
iosxr_content = yaml.safe_load(f)
junos_yaml = os.path.join(YAMLS, 'junos',
'verification_datafile_junos.yaml')
with open(junos_yaml, 'r') as f:
junos_content = yaml.safe_load(f)
# All parser should be in this verification datafile
for osx in self.parsers:
if osx == 'nxos':
os_yaml = nxos_content
os_file = nxos_file
triggers = nxos
elif osx == 'iosxe':
os_yaml = iosxe_content
os_file = iosxe_file
triggers = iosxe
elif osx == 'ios':
os_yaml = ios_content
os_file = ios_file
triggers = ios
elif osx == 'iosxr':
os_yaml = iosxr_content
os_file = iosxr_file
triggers = iosxr
elif osx == 'junos':
os_yaml = junos_content
os_file = junos_file
triggers = junos
for file in self.parsers[osx]:
for parser in self.parsers[osx][file]:
# Check which context exists
for context in CONTEXTS:
if not hasattr(parser, context):
continue
parser_name = parser.__name__
# Verification name
verification_name = 'Verify_{p}'.format(
p=parser_name.replace('Show', ''))
if context != 'cli':
verification_name = '{v}_{c}'.format(
v=verification_name, c=context)
values = []
if parser_name in parser_yaml:
# initial index number
index_num = None
# For all of the combination, add it
# Make the lists ready to go
for key, items in sorted(
parser_yaml[parser_name].items(),
reverse=True):
if isinstance(items, dict):
if key not in parser.__module__:
continue
for ky, val in sorted(items.items(),
reverse=True):
count = 0
if ky == 'zos':
try:
index_num = val.index(osx)
except:
values.append(None)
break
continue
if index_num is not None:
val = val[index_num]
for item in val:
if item == '' or item is None:
count += 1
continue
try:
values[count].extend(
[ky, val[count]])
except IndexError:
values.append([ky, val[count]])
count += 1
else:
count = 0
if key == 'zos':
try:
index_num = items.index(osx)
except:
values.append(None)
break
continue
if index_num is not None:
items = items[index_num]
for item in items:
if item == '' or item is None:
count += 1
continue
try:
values[count].extend(
[key, items[count]])
except IndexError:
values.append([key, items[count]])
count += 1
else:
values.append(None)
for value in values:
if value is not None:
veri_name = '{v}_{e}'.format(
v=verification_name,
e='_'.join(value).replace(' ', '_'))
else:
veri_name = verification_name
main_file[veri_name] = {}
main_file[veri_name]['source'] = {
'class': 'genie.harness.base.Template'
}
main_file[veri_name]['context'] = context
main_file[veri_name]['cmd'] = {}
main_file[veri_name]['cmd'][
'pkg'] = 'genie.libs.parser'
main_file[veri_name]['cmd'][
'class'] = '{f}.{p}'.format(f=file,
p=parser.__name__)
os_file[veri_name] = {}
if veri_name not in EXCLUDE_DEVICES:
os_file[veri_name]['devices'] = ['uut']
if value is not None:
for i in range(0, len(value), 2):
if value[i + 1] != 'default':
if 'parameters' not in os_file[
veri_name]:
os_file[veri_name][
'parameters'] = {}
os_file[veri_name]['parameters'][
value[i]] = value[i + 1]
if veri_name in content:
# Good already exists
# Do not copy source and cmd
# But keep the rest
try:
del content[veri_name]['source']
except:
pass
try:
del content[veri_name]['cmd']
except:
pass
merge_dict(main_file[veri_name],
content[veri_name])
if veri_name in os_yaml:
merge_dict(os_file[veri_name],
os_yaml[veri_name])
triggers.append(veri_name)
# Create the files
with open('verification_datafile.yaml', 'w') as f:
yaml.dump(main_file, f, default_flow_style=False)
self.clean_up('nxos')
with open('nxos/verification_datafile_nxos.yaml', 'w') as f:
yaml.dump(nxos_file, f, default_flow_style=False)
self.clean_up('iosxe')
with open('iosxe/verification_datafile_iosxe.yaml', 'w') as f:
yaml.dump(iosxe_file, f, default_flow_style=False)
self.clean_up('ios')
with open('ios/verification_datafile_ios.yaml', 'w') as f:
yaml.dump(ios_file, f, default_flow_style=False)
self.clean_up('iosxr')
with open('iosxr/verification_datafile_xr.yaml', 'w') as f:
yaml.dump(iosxr_file, f, default_flow_style=False)
self.clean_up('junos')
with open('junos/verification_datafile_junos.yaml', 'w') as f:
yaml.dump(junos_file, f, default_flow_style=False)
log.info(banner('nxos'))
log.info('\n'.join(nxos))
log.info(banner('iosxe'))
log.info('\n'.join(iosxe))
log.info(banner('ios'))
log.info('\n'.join(ios))
log.info(banner('iosxr'))
log.info('\n'.join(iosxr))
log.info(banner('junos'))
log.info('\n'.join(junos))
return main_file