def _load_parser_json(): '''get all parser data in json file''' try: mod = importlib.import_module('genie.libs.parser') parsers = os.path.join(mod.__path__[0], 'parsers.json') except Exception: parsers = '' if not os.path.isfile(parsers): log.warning('parsers.json does not exist, make sure you ' 'are running with latest version of ' 'genie.libs.parsers') parser_data = {} else: # Open all the parsers in json file with open(parsers) as f: parser_data = json.load(f) # check if provided external parser packages ext_parser_package = cfg.get(PYATS_EXT_PARSER, None) or \ os.environ.get(PYATS_EXT_PARSER.upper().replace('.', '_')) if ext_parser_package: ext = ExtendParsers(ext_parser_package) ext.extend() ext.output.pop('tokens', None) summary = ext.output.pop('extend_info', None) merge_dict(parser_data, ext.output, update=True) log.warning("External parser counts: {}\nSummary:\n{}".format( len(summary), json.dumps(summary, indent=2))) return parser_data
def get_policy_map_configurational_policy_map(device, interfaces): """ Get policy-map running configuration Args: device (`obj`): Device object interfaces (`list`): List of interfaces Returns: policy-map configurational dictionary """ out = {} policy_out = {} for interface in interfaces: out[interface] = get_interface_running_config(device, interface) service_policies = [] for item in out[interface]: if interface in item: for service_policy_item in out[interface][item]: if "service-policy" in service_policy_item: service_policies.append(service_policy_item[21:].strip()) for service_policy in service_policies: if "in" in service_policy: direction = "input" else: direction = "output" output = get_policy_map_running_policy_map(device, service_policy) if not output: continue policy_out.setdefault(direction, {}) policy_out[direction] = output for class_name in policy_out[direction]["policy_map"][service_policy][ "class"]: for item in policy_out[direction]["policy_map"][service_policy][ "class"][class_name]: if "service_policy" in item: nested = policy_out[direction]["policy_map"][ service_policy]["class"][class_name][item] nested_policy_out = get_policy_map_running_policy_map( device, nested) if not nested_policy_out: continue new_nested_policy_out = {} new_nested_policy_out.setdefault( "policy_map", {}).setdefault(service_policy, {}).setdefault("child_policy_name", {}) new_nested_policy_out["policy_map"][service_policy][ "child_policy_name"] = nested_policy_out["policy_map"] merge_dict(policy_out[direction], new_nested_policy_out) return policy_out
def load_clean_json(): """get all clean data in json file""" try: mod = importlib.import_module("genie.libs.clean") functions = os.path.join(mod.__path__[0], "clean.json") except Exception: functions = "" if not os.path.isfile(functions): log.warning( "clean.json does not exist, make sure you " "are running with latest version of " "genie.libs.clean" ) clean_data = {} else: # Open all the parsers in json file with open(functions) as f: clean_data = json.load(f) for entry in iter_entry_points(group=CLEAN_PLUGIN_ENTRYPOINT): log.info('Loading clean APIs from {}'.format(entry.module_name)) ext = ExtendClean(entry.module_name) ext.extend() ext.output.pop('tokens', None) log.info("{} clean API count: {}".format( entry.module_name, len(ext.output.keys()))) log.debug('{} clean APIs {}'.format( entry.module_name, json.dumps(ext.output, indent=4) )) plugin_clean_data = ext.output clean_data = merge_dict(clean_data, plugin_clean_data, update=True) return clean_data
def yang_cli(self): cli_output = self.cli() yang_output = self.yang() merged_output = merge_dict(yang_output, cli_output) return merged_output
def learn(self): '''Learn Platform object''' self.add_leaf(cmd=show_platform.ShowInventory, src='[name][(?P<slot>.*)][description]', dest='Slot[(?P<slot>.*)][description]') self.add_leaf(cmd=show_platform.ShowInventory, src='[name][(?P<slot>.*)][serial_number]', dest='Slot[(?P<slot>.*)][serial_number]') self.add_leaf(cmd=show_platform.ShowInventory, src='[name][(?P<slot>.*)][pid]', dest='Slot[(?P<slot>.*)][pid]') self.add_leaf(cmd=show_platform.ShowInventory, src='[name][(?P<slot>.*)][slot]', dest='Slot[(?P<slot>.*)][slot]') # Place holder to make it more readable src_vdc_detail = '[vdc][(?P<id>.*)]' dest_vdc_detail = '[virtual_device][(?P<id>.*)]' self.add_leaf(cmd=show_platform.ShowVdcDetail, src=src_vdc_detail + '[name]', dest=dest_vdc_detail + '[vd_name]') self.add_leaf(cmd=show_platform.ShowVdcDetail, src=src_vdc_detail + '[state]', dest=dest_vdc_detail + '[vd_status]') # Place holder to make it more readable src_vdc_membership = '[virtual_device][(?P<id>.*)][membership]' dest_vdc_membership = '[virtual_device][(?P<id>.*)][membership]' self.add_leaf(cmd=show_platform.ShowVdcMembershipStatus, src=src_vdc_membership + '[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][vd_ms_status]', dest=dest_vdc_membership + '[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][status]') self.add_leaf(cmd=show_platform.ShowVdcMembershipStatus, src=src_vdc_membership + '[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][vd_ms_type]', dest=dest_vdc_membership + '[(?P<vdc_name>.*)][(?P<vd_ms_name>.*)][type]') self.add_leaf(cmd=show_platform.ShowModule, src='[xbar]', dest='[slot][oc]') self.add_leaf(cmd=show_platform.ShowModule, src='[xbar][(?P<xbar>.*)][status]', dest='[slot][oc][(?P<xbar>.*)][state]') self.add_leaf(cmd=show_platform.ShowModule, src='[xbar][(?P<xbar>.*)][module_type]', dest='[slot][oc][(?P<xbar>.*)][name]') self.add_leaf(cmd=show_platform.ShowModule, src='[xbar]', dest='[xbar]') self.add_leaf(cmd=show_platform.ShowModule, src='[slot]', dest='[module]') self.add_leaf(cmd=show_platform.ShowModule, src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][model]', dest='[slot][rp][(?P<slot>.*)][name]') self.add_leaf(cmd=show_platform.ShowModule, src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][status]', dest='[slot][rp][(?P<slot>.*)][state]') self.add_leaf( cmd=show_platform.ShowModule, src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][serial_number]', dest='[slot][rp][(?P<slot>.*)][sn]') self.add_leaf(cmd=show_platform.ShowModule, src='[slot][rp][(?P<slot>.*)][(?P<rp_name>.*)][status]', dest='[slot][rp][(?P<slot>.*)][redundancy_state]') # 'subslot' is not available on NXOS self.add_leaf(cmd=show_platform.Dir, src='[dir]', dest='[dir]') self.add_leaf(cmd=show_platform.Dir, src='[disk_used_space]', dest='[disk_used_space]') self.add_leaf(cmd=show_platform.Dir, src='[disk_free_space]', dest='[disk_free_space]') self.add_leaf(cmd=show_platform.Dir, src='[disk_total_space]', dest='[disk_total_space]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][hardware][memory]', dest='[main_mem]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][software][kickstart]', dest='[kickstart_version]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][software][kickstart_image_file]', dest='[kickstart_image]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][software][system_version]', dest='[version]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][software][system_image_file]', dest='[image]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][os]', dest='[os]') self.add_leaf(cmd=show_platform.ShowVersion, src='[platform][hardware][model]', dest='[rtr_type]') self.add_leaf( cmd=show_platform.ShowInstallActive, src= '[active_packages][active_package_module_0][active_package_name]', dest='[installed_packages]') self.add_leaf(cmd=show_platform.ShowRedundancyStatus, src='[active_supervisor_time]', dest='[rp_uptime]', action=self.convert_to_seconds) self.add_leaf( cmd=show_platform.ShowBoot, src= '[next_reload_boot_variable][sup_number][sup-1][kickstart_variable]', dest='[rp_kickstart_boot_image]') self.add_leaf( cmd=show_platform.ShowBoot, src= '[next_reload_boot_variable][sup_number][sup-1][system_variable]', dest='[rp_boot_image]') self.add_leaf(cmd=show_platform.ShowBoot, src='[next_reload_boot_variable][kickstart_variable]', dest='[rp_kickstart_boot_image]') self.add_leaf(cmd=show_platform.ShowBoot, src='[next_reload_boot_variable][system_variable]', dest='[rp_boot_image]') self.make(final_call=True) if 'Slot' in self.__dict__: line_cards = [] slot_value = None new_dict = {} new_dict['slot'] = {} new_dict['chassis'] = self.Slot['Chassis']['description'].strip() new_dict['chassis_sn'] = self.Slot['Chassis'][ 'serial_number'].strip() for slot in self.Slot: if 'Slot' in slot: slot_value = self.Slot[slot]['slot'] if 'Nexus' in self.Slot[slot]['description']: slot_dict = new_dict.setdefault('slot', {}).setdefault( 'oc', {}).setdefault(slot_value, {}) slot_dict.update( {'name': self.Slot[slot]['description']}) if 'serial_number' in self.Slot[slot]: slot_dict.update( {'sn': self.Slot[slot]['serial_number']}) elif 'Supervisor' not in self.Slot[slot]['description']: slot_dict = new_dict.setdefault('slot', {}).setdefault( 'lc', {}).setdefault(slot_value, {}) slot_dict.update( {'name': self.Slot[slot]['description']}) if 'serial_number' in self.Slot[slot]: slot_dict.update( {'sn': self.Slot[slot]['serial_number']}) elif 'Module' in slot: slot_value = self.Slot[slot]['slot'] if 'Nexus' in self.Slot[slot]['description']: slot_dict = new_dict.setdefault('slot', {}).setdefault( 'oc', {}).setdefault(slot_value, {}) slot_dict.update( {'name': self.Slot[slot]['description']}) slot_dict.update( {'sn': self.Slot[slot]['serial_number']}) if 'Supervisor' not in self.Slot[slot]['description']: slot_dict = new_dict.setdefault('slot', {}).setdefault( 'lc', {}).setdefault(slot_value, {}) slot_dict.update( {'name': self.Slot[slot]['description']}) slot_dict.update( {'sn': self.Slot[slot]['serial_number']}) merge_dict(self.__dict__, new_dict) del self.Slot for slot_number in self.slot['rp']: try: self.slot['rp'][str( slot_number)]['rp_boot_image'] = self.rp_boot_image except Exception: pass try: self.slot['rp'][str( slot_number )]['rp_kickstart_boot_image'] = self.rp_kickstart_boot_image except Exception: pass self.slot['rp'][str(slot_number)]['rp_uptime'] = self.rp_uptime if 'lc' in self.slot: for item in self.slot['lc']: if 'sn' in self.slot['lc'][item]: serial = self.slot['lc'][item]['sn'] else: serial = None if 'module' in self.__dict__: for key in self.module['lc']: for mod_name in self.module['lc'][key]: if 'serial_number' not in self.module['lc'][ key][mod_name]: continue if self.module['lc'][key][mod_name][ 'serial_number'] == serial: linecard_status = self.module['lc'][key][ mod_name]['status'] self.slot['lc'][item].update( {'state': linecard_status}) if 'sup' not in self.slot['lc'][item]['name'].lower() and\ 'ethernet' not in self.slot['lc'][item]['name'].lower(): self.slot.setdefault( 'oc', {}).setdefault(item, {}).update( self.slot['lc'][item]) self.slot['oc'][item].update( {'state': linecard_status}) if 'xbar' in self.__dict__: for key in self.xbar: if 'serial_number' not in self.xbar[key]: continue if self.xbar[key]['serial_number'] == serial: linecard_status = self.xbar[key]['status'] self.slot.setdefault('oc', {}).setdefault( item, {}).update({'state': linecard_status}) if 'oc' in self.slot: for item in self.slot['oc']: if 'sn' in self.slot['oc'][item]: serial = self.slot['oc'][item]['sn'] else: serial = None if 'module' in self.__dict__: for key in self.module['lc']: for mod_name in self.module['lc'][key]: if 'serial_number' not in self.module['lc'][ key][mod_name]: continue if self.module['lc'][key][mod_name][ 'serial_number'] == serial: linecard_status = self.module['lc'][key][ mod_name]['status'] self.slot['oc'][item].update( {'state': linecard_status}) if 'xbar' in self.__dict__: for key in self.xbar: if 'serial_number' not in self.xbar[key]: continue if self.xbar[key]['serial_number'] == serial: linecard_status = self.xbar[key]['status'] self.slot['oc'][item].update( {'state': linecard_status}) if 'oc' in self.slot and 'lc' in self.slot: if item in self.slot['oc'] and item in self.slot['lc']: if self.slot['oc'][item] == self.slot['lc'][item]: self.slot['lc'].pop(item) try: del self.rp_boot_image except Exception: pass try: del self.rp_kickstart_boot_image except Exception: pass del self.module try: del self.xbar except Exception: pass if 'virtual_device' in self.__dict__: for virt_dev in self.virtual_device: if 'membership' in self.virtual_device[virt_dev]: for item in self.virtual_device[virt_dev]['membership']: self.virtual_device[virt_dev]['membership'] = \ self.virtual_device[virt_dev]['membership'][item]
def create_yaml_files(self, datafile): # Load existing Yaml file (If any) # Load main verifications datafiles #main_file = OrderedDict() main_file = {} nxos_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } iosxe_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } ios_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } iosxr_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } junos_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } nxos = [] iosxe = [] ios = [] iosxr = [] junos = [] ios = [] # Load the file with open(datafile, 'r') as f: parser_yaml = yaml.safe_load(f) main_yaml = os.path.join(YAMLS, 'verification_datafile.yaml') with open(main_yaml, 'r') as f: content = yaml.safe_load(f) nxos_yaml = os.path.join(YAMLS, 'nxos', 'verification_datafile_nxos.yaml') with open(nxos_yaml, 'r') as f: nxos_content = yaml.safe_load(f) iosxe_yaml = os.path.join(YAMLS, 'iosxe', 'verification_datafile_iosxe.yaml') with open(iosxe_yaml, 'r') as f: iosxe_content = yaml.safe_load(f) ios_yaml = os.path.join(YAMLS, 'ios', 'verification_datafile_ios.yaml') with open(ios_yaml, 'r') as f: ios_content = yaml.safe_load(f) iosxr_yaml = os.path.join(YAMLS, 'iosxr', 'verification_datafile_xr.yaml') with open(iosxr_yaml, 'r') as f: iosxr_content = yaml.safe_load(f) junos_yaml = os.path.join(YAMLS, 'junos', 'verification_datafile_junos.yaml') with open(junos_yaml, 'r') as f: junos_content = yaml.safe_load(f) # All parser should be in this verification datafile for osx in self.parsers: if osx == 'nxos': os_yaml = nxos_content os_file = nxos_file triggers = nxos elif osx == 'iosxe': os_yaml = iosxe_content os_file = iosxe_file triggers = iosxe elif osx == 'ios': os_yaml = ios_content os_file = ios_file triggers = ios elif osx == 'iosxr': os_yaml = iosxr_content os_file = iosxr_file triggers = iosxr elif osx == 'junos': os_yaml = junos_content os_file = junos_file triggers = junos for file in self.parsers[osx]: for parser in self.parsers[osx][file]: # Check which context exists for context in CONTEXTS: if not hasattr(parser, context): continue parser_name = parser.__name__ # Verification name verification_name = 'Verify_{p}'.format( p=parser_name.replace('Show', '')) if context != 'cli': verification_name = '{v}_{c}'.format( v=verification_name, c=context) values = [] if parser_name in parser_yaml: # initial index number index_num = None # For all of the combination, add it # Make the lists ready to go for key, items in sorted( parser_yaml[parser_name].items(), reverse=True): if isinstance(items, dict): if key not in parser.__module__: continue for ky, val in sorted(items.items(), reverse=True): count = 0 if ky == 'zos': try: index_num = val.index(osx) except: values.append(None) break continue if index_num is not None: val = val[index_num] for item in val: if item == '' or item is None: count += 1 continue try: values[count].extend( [ky, val[count]]) except IndexError: values.append([ky, val[count]]) count += 1 else: count = 0 if key == 'zos': try: index_num = items.index(osx) except: values.append(None) break continue if index_num is not None: items = items[index_num] for item in items: if item == '' or item is None: count += 1 continue try: values[count].extend( [key, items[count]]) except IndexError: values.append([key, items[count]]) count += 1 else: values.append(None) for value in values: if value is not None: veri_name = '{v}_{e}'.format( v=verification_name, e='_'.join(value).replace(' ', '_')) else: veri_name = verification_name main_file[veri_name] = {} main_file[veri_name]['source'] = { 'class': 'genie.harness.base.Template' } main_file[veri_name]['context'] = context main_file[veri_name]['cmd'] = {} main_file[veri_name]['cmd'][ 'pkg'] = 'genie.libs.parser' main_file[veri_name]['cmd'][ 'class'] = '{f}.{p}'.format(f=file, p=parser.__name__) os_file[veri_name] = {} if veri_name not in EXCLUDE_DEVICES: os_file[veri_name]['devices'] = ['uut'] if value is not None: for i in range(0, len(value), 2): if value[i + 1] != 'default': if 'parameters' not in os_file[ veri_name]: os_file[veri_name][ 'parameters'] = {} os_file[veri_name]['parameters'][ value[i]] = value[i + 1] if veri_name in content: # Good already exists # Do not copy source and cmd # But keep the rest try: del content[veri_name]['source'] except: pass try: del content[veri_name]['cmd'] except: pass merge_dict(main_file[veri_name], content[veri_name]) if veri_name in os_yaml: merge_dict(os_file[veri_name], os_yaml[veri_name]) triggers.append(veri_name) # Create the files with open('verification_datafile.yaml', 'w') as f: yaml.dump(main_file, f, default_flow_style=False) self.clean_up('nxos') with open('nxos/verification_datafile_nxos.yaml', 'w') as f: yaml.dump(nxos_file, f, default_flow_style=False) self.clean_up('iosxe') with open('iosxe/verification_datafile_iosxe.yaml', 'w') as f: yaml.dump(iosxe_file, f, default_flow_style=False) self.clean_up('ios') with open('ios/verification_datafile_ios.yaml', 'w') as f: yaml.dump(ios_file, f, default_flow_style=False) self.clean_up('iosxr') with open('iosxr/verification_datafile_xr.yaml', 'w') as f: yaml.dump(iosxr_file, f, default_flow_style=False) self.clean_up('junos') with open('junos/verification_datafile_junos.yaml', 'w') as f: yaml.dump(junos_file, f, default_flow_style=False) log.info(banner('nxos')) log.info('\n'.join(nxos)) log.info(banner('iosxe')) log.info('\n'.join(iosxe)) log.info(banner('ios')) log.info('\n'.join(ios)) log.info(banner('iosxr')) log.info('\n'.join(iosxr)) log.info(banner('junos')) log.info('\n'.join(junos)) return main_file