def parse_imp_inc(self, modules, set_of_names, is_include, schema_part,
capabilities, netconf_version):
for mod in modules:
if is_include:
name = mod.name
conformance_type = 'import'
else:
conformance_type = None
name = mod.arg
if name not in set_of_names:
LOGGER.info('Parsing module {}'.format(name))
set_of_names.add(name)
yang_file = find_first_file('/'.join(self.split[0:-1]),
name + '.yang', name + '@*.yang')
if yang_file is None:
yang_file = find_first_file(self.yang_models_dir,
name + '.yang',
name + '@*.yang')
if yang_file is None:
# TODO add integrity that this file is missing
return
try:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
yang_file,
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
is_vendor_imp_inc=True,
run_integrity=self.run_integrity)
except ParseException as e:
LOGGER.error(e.msg)
continue
yang.parse_all(self.branch, name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(
self.vendor, self.platform_data, self.software_version,
self.os_version, self.feature_set, self.os,
conformance_type, capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker,
self.split, self.os_version)
self.prepare.add_key_sdo_module(yang)
self.parse_imp_inc(yang.submodule, set_of_names, True,
schema_part, capabilities,
netconf_version)
self.parse_imp_inc(yang.imports, set_of_names, False,
schema_part, capabilities,
netconf_version)
except FileNotFoundError:
self.integrity_checker.add_module('/'.join(self.split),
[name])
LOGGER.warning(
'File {} not found in the repository'.format(name))
def parse_and_dump_yang_lib(self):
LOGGER.debug('Starting to parse files from vendor')
capabilities = []
netconf_version = []
LOGGER.debug('Getting capabilities out of api message')
if self.api:
with open(self.hello_message_file.split('.xml')[0] + '.json') as f:
impl = json.load(f)
caps = impl['platforms'].get('netconf-capabilities')
if caps:
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability
capabilities.append(cap_with_version.split('?')[0])
else:
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
if impl['module-list-file'][
'path'] in self.hello_message_file:
caps = impl.get('netconf-capabilities')
if caps:
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability
capabilities.append(
cap_with_version.split('?')[0])
json_file.close()
# netconf capability parsing
modules = self.root[0]
set_of_names = set()
keys = set()
schema_part = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/'
for module in modules:
if 'module-set-id' in module.tag:
continue
LOGGER.debug(
'Getting capabilities out of yang-library xml message')
module_name = None
for mod in module:
if 'name' in mod.tag:
module_name = mod.text
break
yang_lib_info = {}
yang_lib_info['path'] = '/'.join(self.split[0:-1])
yang_lib_info['name'] = module_name
yang_lib_info['features'] = []
yang_lib_info['deviations'] = {}
names = []
revs = []
conformance_type = None
for mod in module:
if 'revision' in mod.tag:
yang_lib_info['revision'] = mod.text
elif 'conformance-type' in mod.tag:
conformance_type = mod.text
elif 'feature' in mod.tag:
yang_lib_info['features'].append(mod.text)
elif 'deviation' in mod.tag:
names.append(mod[0].text)
revs.append(mod[1].text)
yang_lib_info['deviations']['name'] = names
yang_lib_info['deviations']['revision'] = revs
try:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
'/'.join(self.split),
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
True,
True,
yang_lib_info,
run_integrity=self.run_integrity)
except ParseException as e:
LOGGER.error(e.msg)
continue
yang.parse_all(self.branch, module_name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(self.vendor, self.platform_data,
self.software_version,
self.os_version, self.feature_set,
self.os, conformance_type,
capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker, self.split,
self.os_version)
self.prepare.add_key_sdo_module(yang)
keys.add('{}@{}/{}'.format(yang.name, yang.revision,
yang.organization))
set_of_names.add(yang.name)
except FileNotFoundError:
self.integrity_checker.add_module('/'.join(self.split),
[module_name])
LOGGER.warning(
'File {} not found in the repository'.format(module_name))
LOGGER.info('Starting to parse {}'.format(module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capabilities,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capabilities,
netconf_version)
def parse_and_dump(self):
LOGGER.debug('Starting to parse files from vendor')
capabilities = []
tag = self.root.tag
netconf_version = []
# netconf capability parsing
modules = self.root.iter(tag.split('hello')[0] + 'capability')
set_of_names = set()
keys = set()
capabilities_exist = False
LOGGER.debug('Getting capabilities out of hello message')
if os.path.isfile('/'.join(self.split[:-1]) +
'/platform-metadata.json'):
json_file = open('/'.join(self.split[:-1]) +
'/platform-metadata.json')
platforms = json.load(json_file)['platforms']['platform']
for impl in platforms:
if impl['module-list-file']['path'] in self.hello_message_file:
caps = impl.get('netconf-capabilities')
if caps:
capabilities_exist = True
for cap in caps:
capability = cap
# Parse netconf version
if ':netconf:base:' in capability:
netconf_version.append(capability)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
elif ':capability:' in capability:
cap_with_version = capability
capabilities.append(
cap_with_version.split('?')[0])
json_file.close()
LOGGER.debug('Getting capabilities out of hello message')
if not capabilities_exist:
for module in modules:
# Parse netconf version
if ':netconf:base:' in module.text:
netconf_version.append(module.text)
LOGGER.debug('Getting netconf version')
# Parse capability together with version
if ':capability:' in module.text:
cap_with_version = module.text.split(':capability:')[1]
capabilities.append(cap_with_version.split('?')[0])
modules = self.root.iter(tag.split('hello')[0] + 'capability')
schema_part = github_raw + self.owner + '/' + self.repo + '/' +\
self.branch + '/'
# Parse modules
for module in modules:
if 'module=' in module.text:
# Parse name of the module
module_and_more = module.text.split('module=')[1]
module_name = module_and_more.split('&')[0]
LOGGER.info('Parsing module {}'.format(module_name))
try:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
'/'.join(self.split),
self.html_result_dir,
self.parsed_jsons,
self.json_dir,
True,
data=module_and_more,
run_integrity=self.run_integrity)
except ParseException as e:
LOGGER.error(e.msg)
continue
yang.parse_all(self.branch, module_name,
self.prepare.name_revision_organization,
schema_part, self.to)
yang.add_vendor_information(
self.vendor, self.platform_data, self.software_version,
self.os_version, self.feature_set, self.os,
'implement', capabilities, netconf_version,
self.integrity_checker, self.split)
if self.run_integrity:
yang.resolve_integrity(self.integrity_checker,
self.split, self.os_version)
self.prepare.add_key_sdo_module(yang)
key = '{}@{}/{}'.format(yang.name, yang.revision,
yang.organization)
keys.add(key)
set_of_names.add(yang.name)
except FileNotFoundError:
self.integrity_checker.add_module('/'.join(
self.split), [module_and_more.split('&')[0]])
LOGGER.warning(
'File {} not found in the repository'.format(
module_name))
for key in keys:
self.parse_imp_inc(self.prepare.yang_modules[key].submodule,
set_of_names, True, schema_part, capabilities,
netconf_version)
self.parse_imp_inc(self.prepare.yang_modules[key].imports,
set_of_names, False, schema_part, capabilities,
netconf_version)
def parse_and_dump_sdo(self):
if self.api:
LOGGER.debug('Parsing sdo files sent via API')
with open(self.json_dir + '/prepare-sdo.json', 'r') as f:
sdos_json = json.load(f)
sdos_list = sdos_json['modules']['module']
for sdo in sdos_list:
file_name = unicodedata.normalize('NFKD', sdo['source-file']['path'].split('/')[-1]) \
.encode('ascii', 'ignore')
LOGGER.info(
'Parsing sdo file sent via API{}'.format(file_name))
self.owner = sdo['source-file']['owner']
repo_file_path = sdo['source-file']['path']
self.branch = sdo['source-file'].get('branch')
if not self.branch:
self.branch = 'master'
self.repo = sdo['source-file']['repository'].split('.')[0]
root = self.owner + '/' + sdo['source-file']['repository'].split('.')[0] + '/' + self.branch + '/' \
+ '/'.join(repo_file_path.split('/')[:-1])
root = self.json_dir + '/temp/' + root
if sys.version_info < (3, 4):
root = self.json_dir + '/temp/' \
+ unicodedata.normalize('NFKD', root).encode('ascii', 'ignore')
if sys.version_info >= (3, 4):
file_name = file_name.decode('utf-8', 'strict')
if not os.path.isfile(root + '/' + file_name):
LOGGER.error(
'File {} sent via API was not downloaded'.format(
file_name))
continue
if '[1]' not in file_name:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
root + '/' + file_name,
self.html_result_dir, self.parsed_jsons,
self.json_dir)
except ParseException as e:
LOGGER.error(e.msg)
continue
name = file_name.split('.')[0].split('@')[0]
schema = github_raw + self.owner + '/' + self.repo + '/' + self.branch + '/' + repo_file_path
yang.parse_all(self.branch, name,
self.prepare.name_revision_organization,
schema, self.to, sdo)
self.prepare.add_key_sdo_module(yang)
else:
LOGGER.debug('Parsing sdo files from directory')
for root, subdirs, sdos in os.walk('/'.join(self.split)):
for file_name in sdos:
if '.yang' in file_name and ('vendor' not in root
or 'odp' not in root):
LOGGER.info(
'Parsing sdo file from directory {}'.format(
file_name))
if '[1]' in file_name:
LOGGER.warning(
'File {} contains [1] it its file name'.format(
file_name))
else:
try:
yang = Modules(self.yang_models_dir,
self.log_directory,
root + '/' + file_name,
self.html_result_dir,
self.parsed_jsons,
self.json_dir)
except ParseException as e:
LOGGER.error(e.msg)
continue
name = file_name.split('.')[0].split('@')[0]
self.owner = 'YangModels'
self.repo = 'yang'
self.branch = 'master'
path = root + '/' + file_name
abs_path = os.path.abspath(path)
if '/yangmodels/yang/' in abs_path:
path = abs_path.split('/yangmodels/yang/')[1]
else:
path = re.split('tmp\/\w*\/', abs_path)[1]
schema = (github_raw + self.owner + '/' +
self.repo + '/' + self.branch + '/' +
path)
yang.parse_all(
self.branch, name,
self.prepare.name_revision_organization,
schema, self.to)
self.prepare.add_key_sdo_module(yang)