def parse_and_dump(self):
LOGGER.debug('Starting to parse files from vendor')
capability = []
tag = self.root.tag
netconf_version = ''
# netconf capability parsing
modules = self.root.iter(tag.split('hello')[0] + 'capability')
set_of_names = set()
keys = set()
for module in modules:
LOGGER.debug('Getting capabilities out of hello message')
# Parse netconf version
if ':netconf:base:' in module.text:
netconf_version = module.text
LOGGER.debug('Getting netconf version')
# Parse capability together with version
if ':capability:' in module.text:
cap_with_version = module.text.split(':capability:')[1]
capability.append(cap_with_version.split('?')[0])
LOGGER.debug('Getting capabilities out of hello message')
# Parse modules
if 'module=' in module.text:
# Parse name of the module
module_and_more = module.text.split('module=')[1]
module_name = module_and_more.split('&')[0]
LOGGER.info('Parsing module {}'.format(module_name))
try:
yang = Modules('/'.join(self.split),
self.html_result_dir,
self.parsed_jsons,
True,
data=module_and_more)
schema_part = github_raw + self.owner +\
'/' + self.repo + '/' + self.branch + '/'
yang.parse_all(module_name, schema_part)
yang.add_vendor_information(
self.vendor, self.platform_data, self.software_version,
self.os_version, self.feature_set, self.os,
'implement', capability, netconf_version)
yang.resolve_integrity(self.integrity_checker, self.split,
self.os_version)
self.prepare.add_key_sdo_module(yang)
keys.add('{}@{}/{}'.format(yang.name, yang.revision,
yang.organization))
set_of_names.add(yang.name)
except FileError:
self.integrity_checker.add_module('/'.join(
self.split), [module_and_more.split('&')[0]])
LOGGER.warning(
'File {} not found in the repository'.format(
module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capability,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capability,
netconf_version)
def parse_and_dump_sdo(self):
if self.api:
LOGGER.debug('Parsing sdo files sent via API')
with open('../parseAndPopulate/' + self.json_dir + '/prepare-sdo.json', 'r') as f:
sdos_json = json.load(f)
sdos_list = sdos_json['modules']['module']
for sdo in sdos_list:
file_name = unicodedata.normalize('NFKD', sdo['source-file']['path'].split('/')[-1])\
.encode('ascii', 'ignore')
LOGGER.info('Parsing sdo file sent via API{}'.format(file_name))
self.owner = sdo['source-file']['owner']
repo_file_path = sdo['source-file']['path']
self.branch = sdo['source-file'].get('branch')
if not self.branch:
self.branch = 'master'
self.repo = sdo['source-file']['repository'].split('.')[0]
root = self.owner + '/' + sdo['source-file']['repository'].split('.')[0] + '/' + self.branch + '/'\
+ '/'.join(repo_file_path.split('/')[:-1])
root = self.json_dir + '/temp/'\
+ unicodedata.normalize('NFKD', root).encode('ascii', 'ignore')
if not os.path.isfile(root + '/' + file_name):
LOGGER.error('File {} sent via API was not downloaded'.format(file_name))
continue
if '[1]' not in file_name:
yang = Modules(root + '/' + file_name, self.html_result_dir,
self.parsed_jsons, self.json_dir)
name = file_name.split('.')[0].split('@')[0]
schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + repo_file_path
yang.parse_all(name,
self.prepare.name_revision_organization,
schema, self.to, sdo)
self.prepare.add_key_sdo_module(yang)
else:
LOGGER.debug('Parsing sdo files from directory')
for root, subdirs, sdos in os.walk('/'.join(self.split)):
for file_name in sdos:
if '.yang' in file_name and ('vendor' not in root or 'odp' not in root):
LOGGER.info('Parsing sdo file from directory {}'.format(file_name))
if '[1]' in file_name:
LOGGER.warning('File {} contains [1] it its file name'.format(file_name))
else:
yang = Modules(root + '/' + file_name,
self.html_result_dir,
self.parsed_jsons, self.json_dir)
name = file_name.split('.')[0].split('@')[0]
self.owner = 'YangModels'
self.repo = 'yang'
self.branch = 'master'
path = root + '/' + file_name
schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + path.replace(
'../', '')
yang.parse_all(name,
self.prepare.name_revision_organization,
schema, self.to)
self.prepare.add_key_sdo_module(yang)
def parse_imp_inc(self, modules, set_of_names, is_include, schema_part,
capabilities, netconf_version):
for mod in modules:
if is_include:
name = mod.name
conformance_type = 'import'
else:
conformance_type = None
name = mod.arg
if name not in set_of_names:
LOGGER.info('Parsing module {}'.format(name))
set_of_names.add(name)
yang_file = find_first_file('/'.join(self.split[0:-1]),
name + '.yang', name + '@*.yang')
if yang_file is None:
yang_file = find_first_file(
get_curr_dir(__file__) + '/../../.', name + '.yang',
name + '@*.yang')
if yang_file is None:
# TODO add integrity that this file is missing
return
try:
yang = Modules(yang_file,
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
is_vendor_imp_inc=True,
run_integrity=self.run_integrity)
yang.parse_all(name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(
self.vendor, self.platform_data, self.software_version,
self.os_version, self.feature_set, self.os,
conformance_type, capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker,
self.split, self.os_version)
self.prepare.add_key_sdo_module(yang)
self.parse_imp_inc(yang.submodule, set_of_names, True,
schema_part, capabilities,
netconf_version)
self.parse_imp_inc(yang.imports, set_of_names, False,
schema_part, capabilities,
netconf_version)
except FileError:
self.integrity_checker.add_module('/'.join(self.split),
[name])
LOGGER.warning(
'File {} not found in the repository'.format(name))
def parse_and_dump(self):
LOGGER.debug('Starting to parse files from vendor')
capabilities = []
tag = self.root.tag
netconf_version = []
# netconf capability parsing
modules = self.root.iter(tag.split('hello')[0] + 'capability')
set_of_names = set()
keys = set()
capabilities_exist = False
LOGGER.debug('Getting capabilities out of hello message')
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
if impl['module-list-file']['path'] in self.hello_message_file:
caps = impl.get('netconf-capabilities')
if caps:
capabilities_exist = True
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability[1]
capabilities.append(
cap_with_version.split('?')[0])
json_file.close()
LOGGER.debug('Getting capabilities out of hello message')
if not capabilities_exist:
for module in modules:
# Parse netconf version
if ':netconf:base:' in module.text:
netconf_version.append(module.text)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
if ':capability:' in module.text:
cap_with_version = module.text.split(':capability:')[1]
capabilities.append(cap_with_version.split('?')[0])
modules = self.root.iter(tag.split('hello')[0] + 'capability')
schema_part = github_raw + self.owner + '/' + self.repo + '/' +\
self.branch + '/'
# Parse modules
for module in modules:
if 'module=' in module.text:
# Parse name of the module
module_and_more = module.text.split('module=')[1]
module_name = module_and_more.split('&')[0]
LOGGER.info('Parsing module {}'.format(module_name))
try:
yang = Modules('/'.join(self.split),
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
True,
data=module_and_more,
run_integrity=self.run_integrity)
yang.parse_all(module_name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(
self.vendor, self.platform_data, self.software_version,
self.os_version, self.feature_set, self.os,
'implement', capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker,
self.split, self.os_version)
self.prepare.add_key_sdo_module(yang)
key = '{}@{}/{}'.format(yang.name, yang.revision,
yang.organization)
keys.add(key)
set_of_names.add(yang.name)
except FileError:
self.integrity_checker.add_module('/'.join(
self.split), [module_and_more.split('&')[0]])
LOGGER.warning(
'File {} not found in the repository'.format(
module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capabilities,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capabilities,
netconf_version)
def parse_and_dump_yang_lib(self):
LOGGER.debug('Starting to parse files from vendor')
capabilities = []
netconf_version = []
LOGGER.debug('Getting capabilities out of api message')
if self.api:
with open(self.hello_message_file.split('.xml')[0] + '.json') as f:
impl = json.load(f)
caps = impl['platforms'].get('netconf-capabilities')
if caps:
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability[1]
capabilities.append(cap_with_version.split('?')[0])
else:
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
if impl['module-list-file'][
'path'] in self.hello_message_file:
caps = impl.get('netconf-capabilities')
if caps:
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability[1]
capabilities.append(
cap_with_version.split('?')[0])
json_file.close()
# netconf capability parsing
modules = self.root[0]
set_of_names = set()
keys = set()
for module in modules:
if 'module-set-id' in module.tag:
continue
LOGGER.debug(
'Getting capabilities out of yang-library xml message')
module_name = None
for mod in module:
if 'name' in mod.tag:
module_name = mod.text
break
yang_lib_info = {}
yang_lib_info['path'] = '/'.join(self.split[0:-1])
yang_lib_info['name'] = module_name
yang_lib_info['features'] = []
yang_lib_info['deviations'] = {}
names = []
revs = []
conformance_type = None
for mod in module:
if 'revision' in mod.tag:
yang_lib_info['revision'] = mod.text
elif 'conformance-type' in mod.tag:
conformance_type = mod.text
elif 'feature' in mod.tag:
yang_lib_info['features'].append(mod.text)
elif 'deviation' in mod.tag:
names.append(mod[0].text)
revs.append(mod[1].text)
yang_lib_info['deviations']['name'] = names
yang_lib_info['deviations']['revision'] = revs
try:
yang = Modules('/'.join(self.split),
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
True,
True,
yang_lib_info,
run_integrity=self.run_integrity)
schema_part = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/'
yang.parse_all(module_name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(self.vendor, self.platform_data,
self.software_version,
self.os_version, self.feature_set,
self.os, conformance_type,
capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker, self.split,
self.os_version)
self.prepare.add_key_sdo_module(yang)
keys.add('{}@{}/{}'.format(yang.name, yang.revision,
yang.organization))
set_of_names.add(yang.name)
except FileError:
self.integrity_checker.add_module('/'.join(self.split),
[module_name])
LOGGER.warning(
'File {} not found in the repository'.format(module_name))
LOGGER.info('Starting to parse {}'.format(module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capabilities,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capabilities,
netconf_version)
import os
from tools.parseAndPopulate.modules import Modules
from tools.utility.util import get_curr_dir
if __name__ == '__main__':
for root, subdirs, files in os.walk(get_curr_dir(__file__) + '/../../.'):
for f in files:
if f.endswith('.yang'):
name = f.split('@')[0].split('.')[0]
mod = Modules(root + '/' + f, None, None, None)
mod.parse_all(name, 'foo', None, '/home/miroslav/results/')
def parse_and_dump_yang_lib(self):
LOGGER.debug('Starting to parse files from vendor')
capability = []
netconf_version = 'netconf'
# netconf capability parsing
modules = self.root[0]
set_of_names = set()
keys = set()
for module in modules:
if 'module-set-id' in module.tag:
continue
LOGGER.debug(
'Getting capabilities out of yang-library xml message')
module_name = None
for mod in module:
if 'name' in mod.tag:
module_name = mod.text
break
yang_lib_info = {}
yang_lib_info['path'] = '/'.join(self.split[0:-1])
yang_lib_info['name'] = module_name
yang_lib_info['features'] = []
yang_lib_info['deviations'] = {}
names = []
revs = []
conformance_type = None
for mod in module:
if 'revision' in mod.tag:
yang_lib_info['revision'] = mod.text
elif 'conformance-type' in mod.tag:
conformance_type = mod.text
elif 'feature' in mod.tag:
yang_lib_info['features'].append(mod.text)
elif 'deviation' in mod.tag:
names.append(mod[0].text)
revs.append(mod[1].text)
yang_lib_info['deviations']['name'] = names
yang_lib_info['deviations']['revision'] = revs
try:
yang = Modules('/'.join(self.split), self.html_result_dir,
self.parsed_jsons, True, True, yang_lib_info)
schema_part = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/'
yang.parse_all(module_name, schema_part)
yang.add_vendor_information(self.vendor, self.platform_data,
self.software_version,
self.os_version, self.feature_set,
self.os, conformance_type,
capability, netconf_version)
yang.resolve_integrity(self.integrity_checker, self.split,
self.os_version)
self.prepare.add_key_sdo_module(yang)
keys.add('{}@{}/{}'.format(yang.name, yang.revision,
yang.organization))
set_of_names.add(yang.name)
except FileError:
self.integrity_checker.add_module('/'.join(self.split),
[module_name])
LOGGER.warning(
'File {} not found in the repository'.format(module_name))
LOGGER.info('Starting to parse {}'.format(module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capability,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capability,
netconf_version)