Exemple #1
0
    def parse_and_dump(self):
        LOGGER.debug('Starting to parse files from vendor')
        capability = []
        tag = self.root.tag
        netconf_version = ''

        # netconf capability parsing
        modules = self.root.iter(tag.split('hello')[0] + 'capability')
        set_of_names = set()
        keys = set()
        for module in modules:
            LOGGER.debug('Getting capabilities out of hello message')
            # Parse netconf version
            if ':netconf:base:' in module.text:
                netconf_version = module.text
                LOGGER.debug('Getting netconf version')
            # Parse capability together with version
            if ':capability:' in module.text:
                cap_with_version = module.text.split(':capability:')[1]
                capability.append(cap_with_version.split('?')[0])
                LOGGER.debug('Getting capabilities out of hello message')
            # Parse modules
            if 'module=' in module.text:
                # Parse name of the module
                module_and_more = module.text.split('module=')[1]
                module_name = module_and_more.split('&')[0]
                LOGGER.info('Parsing module {}'.format(module_name))
                try:
                    yang = Modules('/'.join(self.split),
                                   self.html_result_dir,
                                   self.parsed_jsons,
                                   True,
                                   data=module_and_more)
                    schema_part = github_raw + self.owner +\
                                  '/' + self.repo + '/' + self.branch + '/'
                    yang.parse_all(module_name, schema_part)
                    yang.add_vendor_information(
                        self.vendor, self.platform_data, self.software_version,
                        self.os_version, self.feature_set, self.os,
                        'implement', capability, netconf_version)
                    yang.resolve_integrity(self.integrity_checker, self.split,
                                           self.os_version)
                    self.prepare.add_key_sdo_module(yang)
                    keys.add('{}@{}/{}'.format(yang.name, yang.revision,
                                               yang.organization))
                    set_of_names.add(yang.name)
                except FileError:
                    self.integrity_checker.add_module('/'.join(
                        self.split), [module_and_more.split('&')[0]])
                    LOGGER.warning(
                        'File {} not found in the repository'.format(
                            module_name))

        for key in keys:
            self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
                               set_of_names, True, schema_part, capability,
                               netconf_version)
            self.parse_imp_inc(self.prepare.yang_modules[key].imports,
                               set_of_names, False, schema_part, capability,
                               netconf_version)
Exemple #2
0
    def parse_and_dump_sdo(self):
        if self.api:
            LOGGER.debug('Parsing sdo files sent via API')
            with open('../parseAndPopulate/' + self.json_dir + '/prepare-sdo.json', 'r') as f:
                sdos_json = json.load(f)
            sdos_list = sdos_json['modules']['module']
            for sdo in sdos_list:
                file_name = unicodedata.normalize('NFKD', sdo['source-file']['path'].split('/')[-1])\
                    .encode('ascii', 'ignore')
                LOGGER.info('Parsing sdo file sent via API{}'.format(file_name))
                self.owner = sdo['source-file']['owner']
                repo_file_path = sdo['source-file']['path']
                self.branch = sdo['source-file'].get('branch')
                if not self.branch:
                    self.branch = 'master'
                self.repo = sdo['source-file']['repository'].split('.')[0]
                root = self.owner + '/' + sdo['source-file']['repository'].split('.')[0] + '/' + self.branch + '/'\
                    + '/'.join(repo_file_path.split('/')[:-1])
                root = self.json_dir + '/temp/'\
                    + unicodedata.normalize('NFKD', root).encode('ascii', 'ignore')
                if not os.path.isfile(root + '/' + file_name):
                    LOGGER.error('File {} sent via API was not downloaded'.format(file_name))
                    continue
                if '[1]' not in file_name:
                    yang = Modules(root + '/' + file_name, self.html_result_dir,
                                   self.parsed_jsons, self.json_dir)
                    name = file_name.split('.')[0].split('@')[0]
                    schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + repo_file_path
                    yang.parse_all(name,
                                   self.prepare.name_revision_organization,
                                   schema, self.to, sdo)
                    self.prepare.add_key_sdo_module(yang)

        else:
            LOGGER.debug('Parsing sdo files from directory')
            for root, subdirs, sdos in os.walk('/'.join(self.split)):
                for file_name in sdos:
                    if '.yang' in file_name and ('vendor' not in root or 'odp' not in root):
                        LOGGER.info('Parsing sdo file from directory {}'.format(file_name))

                        if '[1]' in file_name:
                            LOGGER.warning('File {} contains [1] it its file name'.format(file_name))
                        else:
                            yang = Modules(root + '/' + file_name,
                                           self.html_result_dir,
                                           self.parsed_jsons, self.json_dir)
                            name = file_name.split('.')[0].split('@')[0]
                            self.owner = 'YangModels'
                            self.repo = 'yang'
                            self.branch = 'master'
                            path = root + '/' + file_name
                            schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + path.replace(
                                '../', '')
                            yang.parse_all(name,
                                           self.prepare.name_revision_organization,
                                           schema, self.to)
                            self.prepare.add_key_sdo_module(yang)
Exemple #3
0
 def parse_imp_inc(self, modules, set_of_names, is_include, schema_part,
                   capabilities, netconf_version):
     for mod in modules:
         if is_include:
             name = mod.name
             conformance_type = 'import'
         else:
             conformance_type = None
             name = mod.arg
         if name not in set_of_names:
             LOGGER.info('Parsing module {}'.format(name))
             set_of_names.add(name)
             yang_file = find_first_file('/'.join(self.split[0:-1]),
                                         name + '.yang', name + '@*.yang')
             if yang_file is None:
                 yang_file = find_first_file(
                     get_curr_dir(__file__) + '/../../.', name + '.yang',
                     name + '@*.yang')
             if yang_file is None:
                 # TODO add integrity that this file is missing
                 return
             try:
                 yang = Modules(yang_file,
                                self.html_result_dir,
                                self.parsed_jsons,
                                self.json_dir,
                                is_vendor_imp_inc=True,
                                run_integrity=self.run_integrity)
                 yang.parse_all(name,
                                self.prepare.name_revision_organization,
                                schema_part, self.to)
                 yang.add_vendor_information(
                     self.vendor, self.platform_data, self.software_version,
                     self.os_version, self.feature_set, self.os,
                     conformance_type, capabilities, netconf_version,
                     self.integrity_checker, self.split)
                 if self.run_integrity:
                     yang.resolve_integrity(self.integrity_checker,
                                            self.split, self.os_version)
                 self.prepare.add_key_sdo_module(yang)
                 self.parse_imp_inc(yang.submodule, set_of_names, True,
                                    schema_part, capabilities,
                                    netconf_version)
                 self.parse_imp_inc(yang.imports, set_of_names, False,
                                    schema_part, capabilities,
                                    netconf_version)
             except FileError:
                 self.integrity_checker.add_module('/'.join(self.split),
                                                   [name])
                 LOGGER.warning(
                     'File {} not found in the repository'.format(name))
Exemple #4
0
    def parse_and_dump(self):
        LOGGER.debug('Starting to parse files from vendor')
        capabilities = []
        tag = self.root.tag
        netconf_version = []

        # netconf capability parsing
        modules = self.root.iter(tag.split('hello')[0] + 'capability')
        set_of_names = set()
        keys = set()
        capabilities_exist = False
        LOGGER.debug('Getting capabilities out of hello message')
        if os.path.isfile('/'.join(self.split[:-1]) +
                          '/platform-metadata.json'):
            json_file = open('/'.join(self.split[:-1]) +
                             '/platform-metadata.json')
            platforms = json.load(json_file)['platforms']['platform']
            for impl in platforms:
                if impl['module-list-file']['path'] in self.hello_message_file:
                    caps = impl.get('netconf-capabilities')
                    if caps:
                        capabilities_exist = True
                        for cap in caps:
                            capability = cap
                            # Parse netconf version
                            if ':netconf:base:' in capability:
                                netconf_version.append(capability)
                                LOGGER.debug('Getting netconf version')
                            # Parse capability together with version
                            elif ':capability:' in capability:
                                cap_with_version = capability[1]
                                capabilities.append(
                                    cap_with_version.split('?')[0])
            json_file.close()
        LOGGER.debug('Getting capabilities out of hello message')
        if not capabilities_exist:
            for module in modules:
                # Parse netconf version
                if ':netconf:base:' in module.text:
                    netconf_version.append(module.text)
                    LOGGER.debug('Getting netconf version')
                # Parse capability together with version
                if ':capability:' in module.text:
                    cap_with_version = module.text.split(':capability:')[1]
                    capabilities.append(cap_with_version.split('?')[0])
            modules = self.root.iter(tag.split('hello')[0] + 'capability')

        schema_part = github_raw + self.owner + '/' + self.repo + '/' +\
                      self.branch + '/'
        # Parse modules
        for module in modules:
            if 'module=' in module.text:
                # Parse name of the module
                module_and_more = module.text.split('module=')[1]
                module_name = module_and_more.split('&')[0]
                LOGGER.info('Parsing module {}'.format(module_name))
                try:
                    yang = Modules('/'.join(self.split),
                                   self.html_result_dir,
                                   self.parsed_jsons,
                                   self.json_dir,
                                   True,
                                   data=module_and_more,
                                   run_integrity=self.run_integrity)

                    yang.parse_all(module_name,
                                   self.prepare.name_revision_organization,
                                   schema_part, self.to)
                    yang.add_vendor_information(
                        self.vendor, self.platform_data, self.software_version,
                        self.os_version, self.feature_set, self.os,
                        'implement', capabilities, netconf_version,
                        self.integrity_checker, self.split)
                    if self.run_integrity:
                        yang.resolve_integrity(self.integrity_checker,
                                               self.split, self.os_version)
                    self.prepare.add_key_sdo_module(yang)
                    key = '{}@{}/{}'.format(yang.name, yang.revision,
                                            yang.organization)
                    keys.add(key)
                    set_of_names.add(yang.name)
                except FileError:
                    self.integrity_checker.add_module('/'.join(
                        self.split), [module_and_more.split('&')[0]])
                    LOGGER.warning(
                        'File {} not found in the repository'.format(
                            module_name))

        for key in keys:
            self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
                               set_of_names, True, schema_part, capabilities,
                               netconf_version)
            self.parse_imp_inc(self.prepare.yang_modules[key].imports,
                               set_of_names, False, schema_part, capabilities,
                               netconf_version)
Exemple #5
0
    def parse_and_dump_yang_lib(self):
        LOGGER.debug('Starting to parse files from vendor')
        capabilities = []
        netconf_version = []
        LOGGER.debug('Getting capabilities out of api message')
        if self.api:
            with open(self.hello_message_file.split('.xml')[0] + '.json') as f:
                impl = json.load(f)
            caps = impl['platforms'].get('netconf-capabilities')
            if caps:
                for cap in caps:
                    capability = cap
                    # Parse netconf version
                    if ':netconf:base:' in capability:
                        netconf_version.append(capability)
                        LOGGER.debug('Getting netconf version')
                    # Parse capability together with version
                    elif ':capability:' in capability:
                        cap_with_version = capability[1]
                        capabilities.append(cap_with_version.split('?')[0])
        else:
            if os.path.isfile('/'.join(self.split[:-1]) +
                              '/platform-metadata.json'):
                json_file = open('/'.join(self.split[:-1]) +
                                 '/platform-metadata.json')
                platforms = json.load(json_file)['platforms']['platform']
                for impl in platforms:
                    if impl['module-list-file'][
                            'path'] in self.hello_message_file:
                        caps = impl.get('netconf-capabilities')
                        if caps:
                            for cap in caps:
                                capability = cap
                                # Parse netconf version
                                if ':netconf:base:' in capability:
                                    netconf_version.append(capability)
                                    LOGGER.debug('Getting netconf version')
                                # Parse capability together with version
                                elif ':capability:' in capability:
                                    cap_with_version = capability[1]
                                    capabilities.append(
                                        cap_with_version.split('?')[0])
                json_file.close()

        # netconf capability parsing
        modules = self.root[0]
        set_of_names = set()
        keys = set()
        for module in modules:
            if 'module-set-id' in module.tag:
                continue
            LOGGER.debug(
                'Getting capabilities out of yang-library xml message')
            module_name = None

            for mod in module:
                if 'name' in mod.tag:
                    module_name = mod.text
                    break

            yang_lib_info = {}
            yang_lib_info['path'] = '/'.join(self.split[0:-1])
            yang_lib_info['name'] = module_name
            yang_lib_info['features'] = []
            yang_lib_info['deviations'] = {}
            names = []
            revs = []
            conformance_type = None
            for mod in module:
                if 'revision' in mod.tag:
                    yang_lib_info['revision'] = mod.text
                elif 'conformance-type' in mod.tag:
                    conformance_type = mod.text
                elif 'feature' in mod.tag:
                    yang_lib_info['features'].append(mod.text)
                elif 'deviation' in mod.tag:
                    names.append(mod[0].text)
                    revs.append(mod[1].text)
            yang_lib_info['deviations']['name'] = names
            yang_lib_info['deviations']['revision'] = revs

            try:
                yang = Modules('/'.join(self.split),
                               self.html_result_dir,
                               self.parsed_jsons,
                               self.json_dir,
                               True,
                               True,
                               yang_lib_info,
                               run_integrity=self.run_integrity)
                schema_part = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/'
                yang.parse_all(module_name,
                               self.prepare.name_revision_organization,
                               schema_part, self.to)
                yang.add_vendor_information(self.vendor, self.platform_data,
                                            self.software_version,
                                            self.os_version, self.feature_set,
                                            self.os, conformance_type,
                                            capabilities, netconf_version,
                                            self.integrity_checker, self.split)
                if self.run_integrity:
                    yang.resolve_integrity(self.integrity_checker, self.split,
                                           self.os_version)
                self.prepare.add_key_sdo_module(yang)
                keys.add('{}@{}/{}'.format(yang.name, yang.revision,
                                           yang.organization))
                set_of_names.add(yang.name)
            except FileError:
                self.integrity_checker.add_module('/'.join(self.split),
                                                  [module_name])
                LOGGER.warning(
                    'File {} not found in the repository'.format(module_name))

            LOGGER.info('Starting to parse {}'.format(module_name))

        for key in keys:
            self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
                               set_of_names, True, schema_part, capabilities,
                               netconf_version)
            self.parse_imp_inc(self.prepare.yang_modules[key].imports,
                               set_of_names, False, schema_part, capabilities,
                               netconf_version)
import os

from tools.parseAndPopulate.modules import Modules
from tools.utility.util import get_curr_dir

if __name__ == '__main__':
    for root, subdirs, files in os.walk(get_curr_dir(__file__) + '/../../.'):
        for f in files:
            if f.endswith('.yang'):
                name = f.split('@')[0].split('.')[0]
                mod = Modules(root + '/' + f, None, None, None)
                mod.parse_all(name, 'foo', None, '/home/miroslav/results/')
Exemple #7
0
    def parse_and_dump_yang_lib(self):
        LOGGER.debug('Starting to parse files from vendor')
        capability = []
        netconf_version = 'netconf'

        # netconf capability parsing
        modules = self.root[0]
        set_of_names = set()
        keys = set()
        for module in modules:
            if 'module-set-id' in module.tag:
                continue
            LOGGER.debug(
                'Getting capabilities out of yang-library xml message')
            module_name = None

            for mod in module:
                if 'name' in mod.tag:
                    module_name = mod.text
                    break

            yang_lib_info = {}
            yang_lib_info['path'] = '/'.join(self.split[0:-1])
            yang_lib_info['name'] = module_name
            yang_lib_info['features'] = []
            yang_lib_info['deviations'] = {}
            names = []
            revs = []
            conformance_type = None
            for mod in module:
                if 'revision' in mod.tag:
                    yang_lib_info['revision'] = mod.text
                elif 'conformance-type' in mod.tag:
                    conformance_type = mod.text
                elif 'feature' in mod.tag:
                    yang_lib_info['features'].append(mod.text)
                elif 'deviation' in mod.tag:
                    names.append(mod[0].text)
                    revs.append(mod[1].text)
            yang_lib_info['deviations']['name'] = names
            yang_lib_info['deviations']['revision'] = revs

            try:
                yang = Modules('/'.join(self.split), self.html_result_dir,
                               self.parsed_jsons, True, True, yang_lib_info)
                schema_part = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/'
                yang.parse_all(module_name, schema_part)
                yang.add_vendor_information(self.vendor, self.platform_data,
                                            self.software_version,
                                            self.os_version, self.feature_set,
                                            self.os, conformance_type,
                                            capability, netconf_version)
                yang.resolve_integrity(self.integrity_checker, self.split,
                                       self.os_version)
                self.prepare.add_key_sdo_module(yang)
                keys.add('{}@{}/{}'.format(yang.name, yang.revision,
                                           yang.organization))
                set_of_names.add(yang.name)
            except FileError:
                self.integrity_checker.add_module('/'.join(self.split),
                                                  [module_name])
                LOGGER.warning(
                    'File {} not found in the repository'.format(module_name))

            LOGGER.info('Starting to parse {}'.format(module_name))

        for key in keys:
            self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
                               set_of_names, True, schema_part, capability,
                               netconf_version)
            self.parse_imp_inc(self.prepare.yang_modules[key].imports,
                               set_of_names, False, schema_part, capability,
                               netconf_version)