def test_check_depencendies(self):
good = parse(self.module_path('good'))
assert good
all_imports, missing_imports = itg.check_dependencies('import', good, self.module_dir, self.yang_models)
self.assertSetEqual(all_imports, {'invalid-revision'})
self.assertFalse(missing_imports)
all_includes, missing_includes = itg.check_dependencies('include', good, self.module_dir, self.yang_models)
self.assertSetEqual(all_includes, {'invalid-namespace', 'l1-dependency'})
self.assertFalse(missing_includes)
missing_import = parse(self.module_path('missing-import'))
assert missing_import
all_imports, missing_imports = itg.check_dependencies('import', missing_import,
self.module_dir, self.yang_models)
self.assertSetEqual(all_imports, {'nonexistent'})
self.assertSetEqual(missing_imports, {'nonexistent'})
missing_include = parse(self.module_path('missing-include'))
assert missing_include
all_includes, missing_includes = itg.check_dependencies('include', missing_include,
self.module_dir, self.yang_models)
self.assertSetEqual(all_includes, {'nonexistent'})
self.assertSetEqual(missing_includes, {'nonexistent'})
def test_check_revision(self):
good = parse(self.module_path('good'))
assert good
self.assertTrue(itg.check_revision(good))
missing_revision = parse(self.module_path('missing-revision'))
assert missing_revision
self.assertFalse(itg.check_revision(missing_revision))
invalid_revision = parse(self.module_path('invalid-revision'))
assert invalid_revision
self.assertFalse(itg.check_revision(invalid_revision))
def test_check_namespace(self):
good = parse(self.module_path('good'))
assert good
self.assertTrue(itg.check_namespace(good))
missing_namespace = parse(self.module_path('missing-namespace'))
assert missing_namespace
self.assertFalse(itg.check_namespace(missing_namespace))
invalid_namespace = parse(self.module_path('invalid-namespace'))
assert invalid_namespace
self.assertFalse(itg.check_namespace(invalid_namespace))
def get_available_commit_hash(module: dict, commit_hash_list: list):
name = module.get('name')
schema = module.get('schema', '')
revision = module.get('revision')
branch = get_branch_from_schema(schema)
available_commit_hash = ''
for commit_hash in commit_hash_list:
new_schema = schema.replace(branch, commit_hash)
if new_schema not in requests_done:
requests_done.append(new_schema)
response = requests.get(new_schema)
if response.status_code == 200:
module_text = response.text
try:
module_revision = yangParser.parse(module_text).search(
'revision')[0].arg
except:
module_revision = '1970-01-01'
key = '{}@{}'.format(name, module_revision)
available_schemas[key] = new_schema
if revision == module_revision:
available_commit_hash = commit_hash
break
return available_commit_hash
def find_first_file(directory, pattern, pattern_with_revision):
"""Find first yang file based on name or name and revision
:param directory: (str) directory where to look
for a file
:param pattern: (str) name of the yang file
:param pattern_with_revision: name and revision
of the in format <name>@<revision>
:return path to current directory
"""
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern_with_revision):
filename = os.path.join(root, basename)
return filename
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern):
filename = os.path.join(root, basename)
try:
revision = yangParser.parse(filename).search('revision')[0]\
.arg
except:
revision = '1970-01-01'
if '*' not in pattern_with_revision:
if revision in pattern_with_revision:
return filename
else:
return filename
def get_specifics(path_dir):
"""Get amount of yang files in specified directory and the amount that
passed compilation
Arguments:
:param path_dir: (str) path to directory where we are searching for
yang files
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
passed = 0
num_in_catalog = 0
yang_modules = list_of_yang_modules_in_subdir(path_dir)
yang_modules_length = len(yang_modules)
x = 0
for mod_git in yang_modules:
x += 1
LOGGER.info("{} out of {} getting specifics from {}".format(x, yang_modules_length, path_dir))
try:
parsed_yang = yangParser.parse(os.path.abspath(mod_git))
revision = parsed_yang.search('revision')[0].arg
except:
continue
organization = resolve_organization(mod_git, parsed_yang)
name = mod_git.split('/')[-1].split('.')[0].split('@')[0]
if revision is None:
revision = '1970-01-01'
if name is None or organization is None:
continue
if ',' in organization:
organization = organization.replace(' ', '%20')
path = '{}search/name/{}'.format(yangcatalog_api_prefix, name)
module_exist = requests.get(path, auth=(auth[0], auth[1]),
headers={'Accept': 'application/vnd.yang.data+json',
'Content-Type': 'application/vnd.yang.data+json'})
if repr(module_exist.status_code).startswith('20'):
data = module_exist.json()
org = data['yang-catalog:modules']['module'][0]['organization']
rev = data['yang-catalog:modules']['module'][0]['revision']
status = data['yang-catalog:modules']['module'][0].get('compilation-status')
if org == organization and rev == revision:
if 'passed' == status:
passed += 1
num_in_catalog += 1
else:
LOGGER.error('Could not send request with path {}'.format(path))
else:
organization = organization.replace(' ', '%20')
mod = '{}@{}_{}'.format(name, revision, organization)
data = all_modules_data_unique.get(mod)
if data is not None:
if 'passed' == data.get('compilation-status'):
passed += 1
num_in_catalog += 1
else:
LOGGER.error('module {} does not exist'.format(mod))
return [num_in_catalog, passed]
def resolve_revision(yang_file):
"""
search for revision in file "yang_file"
:param yang_file: yang file
:return: revision of the yang file
"""
try:
parsed_yang = yangParser.parse(os.path.abspath(yang_file))
revision = parsed_yang.search('revision')[0].arg
except:
revision = '1970-01-01'
return revision
def __init__(self,
name: str,
path: str,
jsons: LoadFiles,
dir_paths: DirPaths,
git_commit_hash: str,
yang_modules: dict,
schema_base: str,
aditional_info: t.Optional[t.Dict[str, str]] = None,
submodule_name: t.Optional[str] = None,
data: t.Optional[t.Union[str, dict]] = None):
real_path = path
# these are required for self._find_file() to work
self.yang_models = dir_paths['yang_models']
self._path = path
self.features = []
self.deviations = []
if isinstance(data, str): # string from a capabilities file
self.features = self._resolve_deviations_and_features(
'features=', data)
deviation_names = self._resolve_deviations_and_features(
'deviations=', data)
for name in deviation_names:
deviation = {'name': name}
yang_file = self._find_file(name)
if yang_file is None:
deviation['revision'] = '1970-01-01'
else:
try:
deviation['revision'] = yangParser.parse(os.path.abspath(yang_file)) \
.search('revision')[0].arg
except:
deviation['revision'] = '1970-01-01'
self.deviations.append(deviation)
if 'revision' in data:
revision_and_more = data.split('revision=')[1]
revision = revision_and_more.split('&')[0]
self.revision = revision
real_path = self._find_file(data.split('&')[0], self.revision)
elif isinstance(data,
dict): # dict parsed out from a ietf-yang-library file
self.deviations = data['deviations']
self.features = data['features']
self.revision = data['revision']
real_path = self._find_file(data['name'], self.revision)
if not real_path:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
path)
super().__init__(name, os.path.abspath(real_path), jsons, dir_paths,
git_commit_hash, yang_modules, schema_base,
aditional_info, submodule_name)
def resolve_revision(yang_file: str):
""" Search for revision in file "yang_file"
Argument:
:param yang_file (str) full path to the yang file
:return: revision of the yang file
"""
try:
parsed_yang = yangParser.parse(os.path.abspath(yang_file))
revision = parsed_yang.search('revision')[0].arg
except Exception:
revision = '1970-01-01'
return revision
def get_latest_revision(f, LOGGER):
"""
Search for revision in yang file
:param f: yang file
:return: revision of the file "f"
"""
stmt = yangParser.parse(f)
if stmt is None: # In case of invalid YANG syntax, None is returned
LOGGER.info('Cannot yangParser.parse ' + f)
return None
rev = stmt.search_one('revision')
if rev is None:
return None
return rev.arg
def __resolve_submodule(self):
try:
submodules = self.__parsed_yang.search('include')
except:
return
if len(submodules) == 0:
return
for chunk in submodules:
dep = self.Dependencies()
sub = self.Submodules()
sub.name = chunk.arg
if len(chunk.search('revision-date')) > 0:
sub.revision = chunk.search('revision-date')[0].arg
if sub.revision:
yang_file = self.__find_file(sub.name, sub.revision, True)
dep.revision = sub.revision
else:
yang_file = self.__find_file(sub.name, submodule=True)
try:
sub.revision = \
yangParser.parse(os.path.abspath(yang_file)).search(
'revision')[0].arg
except:
sub.revision = '1970-01-01'
if yang_file is None:
LOGGER.error('Module can not be found')
continue
path = '/'.join(self.schema.split('/')[0:-1])
path += '/{}'.format(yang_file.split('/')[-1])
if yang_file:
sub.schema = path
dep.name = sub.name
dep.schema = sub.schema
self.dependencies.append(dep)
self.submodule.append(sub)
self.json_submodules = json.dumps([{
'name':
self.submodule[x].name,
'schema':
self.submodule[x].schema,
'revision':
self.submodule[x].revision
} for x in range(0, len(self.submodule))])
def find_first_file(directory: str, pattern: str,
pattern_with_revision: str, yang_models_dir: str = '') -> t.Optional[str]:
""" Search for the first file in 'directory' which either match 'pattern' or 'pattern_with_revision' string.
Arguments:
:param directory (str) directory where to look for a file
:param pattern (str) name of the yang file
:param pattern_with_revision (str) name and revision of the module in format <name>@<revision>
:param yang_models_dir (str) path to the directory where YangModels/yang repo is cloned
:return (str) path to matched file
"""
def match_file(directory, pattern) -> t.Optional[str]:
for root, _, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern):
return os.path.join(root, basename)
rfcs_dir = '{}/standard/ietf/RFC'.format(yang_models_dir)
standards_dir = '{}/standard'.format(yang_models_dir)
experimental_dir = '{}/experimental/ietf-extracted-YANG-modules'.format(yang_models_dir)
paths_to_check = [directory, rfcs_dir, standards_dir, experimental_dir, yang_models_dir]
patterns_order = []
if '*' not in pattern_with_revision:
patterns_order = [pattern_with_revision, pattern]
else:
patterns_order = [pattern, pattern_with_revision]
for path in paths_to_check:
for pattern in patterns_order:
filename = match_file(path, pattern)
if filename:
try:
parsed = yangParser.parse(filename)
except:
continue
results = parsed.search('revision')
if results:
revision = results[0].arg
else:
revision = '1970-01-01'
if '*' not in pattern_with_revision:
if revision in pattern_with_revision:
return filename
else:
return filename
def _resolve_submodule(self, dependencies: t.List[Dependency],
submodule: t.List[Submodules]):
LOGGER.debug('Resolving submodules')
try:
submodules = self._parsed_yang.search('include')
except:
return
if len(submodules) == 0:
return
for chunk in submodules:
dep = Dependency()
sub = Submodules()
sub.name = chunk.arg
if len(chunk.search('revision-date')) > 0:
sub.revision = chunk.search('revision-date')[0].arg
if sub.revision:
yang_file = self._find_file(sub.name, sub.revision)
dep.revision = sub.revision
else:
yang_file = self._find_file(sub.name)
if yang_file:
try:
sub.revision = \
yangParser.parse(os.path.abspath(yang_file)).search('revision')[0].arg
except:
sub.revision = '1970-01-01'
else:
sub.revision = '1970-01-01'
if yang_file is None:
LOGGER.error('Module can not be found')
continue
if self.schema:
sub_schema = '/'.join(
(*self.schema.split('/')[0:-1], yang_file.split('/')[-1]))
else:
sub_schema = None
if yang_file:
sub.schema = sub_schema
dep.name = sub.name
dep.schema = sub.schema
dependencies.append(dep)
submodule.append(sub)
def resolve_organization(path: str, parsed_yang) -> str:
"""Parse yang file and resolve organization out of the module. If the module
is a submodule find it's parent and resolve its organization
Arguments:
:param path: (str) path to a file to parse and resolve a organization
:param parsed_yang (object) pyang parsed yang file object
:return: (str) organization the yang file belongs to
"""
organization = None
results = parsed_yang.search('organization')
if results:
result = results[0].arg.lower()
if 'cisco' in result:
organization = 'cisco'
elif 'ietf' in result:
organization = 'ietf'
namespace = None
results = parsed_yang.search('namespace')
if results:
namespace = results[0].arg.lower()
else:
results = parsed_yang.search('belongs-to')
if results:
belongs_to = results[0].arg
pattern = '{}.yang'.format(belongs_to)
pattern_with_revision = '{}@*.yang'.format(belongs_to)
results = None
yang_file = find_first_file(os.path.dirname(path), pattern,
pattern_with_revision)
if yang_file is None:
yang_file = find_first_file('/'.join(path.split('/')[:-2]),
pattern, pattern_with_revision)
if yang_file is not None:
try:
parsed = yangParser.parse(os.path.abspath(yang_file))
if parsed:
results = parsed.search('namespace')
if results:
namespace = results[0].arg.lower()
except yangParser.ParseException:
pass
if namespace is None:
return MISSING_ELEMENT
else:
return match_organization(namespace, organization)
def get_latest_revision(path: str, LOGGER: logging.Logger):
""" Search for the latest revision in yang file
Arguments:
:param path (str) full path to the yang file
:param LOGGER (logging.Logger) formated logger with the specified name
:return revision of the module at the given path
"""
try:
stmt = yangParser.parse(path)
result = stmt.search_one('revision')
assert result
rev = result.arg
except Exception:
LOGGER.error('Cannot yangParser.parse {}'.format(path))
rev = None # In case of invalid YANG syntax, None is returned
return rev
def add_vendor_information(self, vendor, platform_data, software_version,
os_version, feature_set, os_type,
confarmance_type, capability, netconf_version,
integrity_checker, split):
for data in platform_data:
implementation = self.Implementations()
implementation.vendor = vendor
implementation.platform = data['platform']
implementation.software_version = software_version
implementation.software_flavor = data['software-flavor']
implementation.os_version = os_version
implementation.feature_set = feature_set
implementation.os_type = os_type
implementation.feature = self.features
implementation.capability = capability
implementation.netconf_version = netconf_version
if self.is_yang_lib:
for deviation in self.deviations:
devs = implementation.Deviations()
devs.name = deviation['name']
devs.revision = deviation['revision']
implementation.deviations.append(devs)
else:
for name in self.deviations:
devs = implementation.Deviations()
devs.name = name
yang_file = self.__find_file(name)
if yang_file is None:
devs.revision = '1970-01-01'
else:
try:
s = yang_file.split('/')
key = '/'.join(split[0:-1])
integrity_checker.remove_one(key, s[-1])
devs.revision = yangParser.parse(os.path.abspath(yang_file)) \
.search('revision')[0].arg
except:
devs.revision = '1970-01-01'
implementation.deviations.append(devs)
implementation.conformance_type = confarmance_type
self.implementation.append(implementation)
def check(path: str, directory: str, yang_models_dir: str, sdo: bool):
try:
parsed_module = yangParser.parse(path)
except yangParser.ParseException:
return
if not check_revision(parsed_module):
missing_revisions.add(path)
if not check_namespace(parsed_module):
missing_namespaces.add(path)
all_imports, broken_imports = check_dependencies('import', parsed_module, directory, yang_models_dir)
if broken_imports:
missing_modules[path] |= broken_imports
all_includes, broken_includes = check_dependencies('include', parsed_module, directory, yang_models_dir)
if broken_includes:
missing_submodules[path] |= broken_includes
if not sdo:
all_dependencies = all_imports | all_includes
for module in all_dependencies:
if module in unused_modules[directory]:
unused_modules[directory].remove(module)
modules_to_check.append(module)
def _resolve_submodule_case(self, field) -> t.Optional[str]:
if self.module_type == 'submodule':
LOGGER.debug(
'Getting parent information because file {} is a submodule'.
format(self._path))
if self.belongs_to:
yang_file = self._find_file(self.belongs_to)
else:
return None
if yang_file is None:
return None
try:
parsed_parent_yang = yangParser.parse(
os.path.abspath(yang_file))
return parsed_parent_yang.search(field)[0].arg
except:
return None
else:
try:
return self._parsed_yang.search(field)[0].arg
except:
return None
def __init__(self, name: str, path: str, jsons: LoadFiles,
dir_paths: DirPaths, git_commit_hash: str, yang_modules: dict,
schema_base: str, aditional_info: t.Optional[t.Dict[str,
str]],
submodule_name: t.Optional[str]):
"""
Initialize and parse everything out of a module.
Arguments:
:param name: (str) name of the module (not parsed out of the module)
:param path: (str) path to yang file being parsed
:param jsons: (obj) LoadFiles class containing all the json
and html files with parsed results
:param dir_paths: (dict) paths to various needed directories according to configuration
:param git_commit_hash: (str) name of the git commit hash where we can find the module
:param yang_modules: (dict) yang modules we've already parsed
:param schema_base: (str) url to a raw module on github up to and not including the
path of the file in the repo
:param aditional_info: (dict) some aditional information about module given from client
:param submodule_name: (str) name of the git submodule the yang module belongs to
"""
global LOGGER
LOGGER = log.get_logger(
'modules', '{}/parseAndPopulate.log'.format(dir_paths['log']))
config = create_config()
self._web_uri = config.get('Web-Section',
'my-uri',
fallback='https://yangcatalog.org')
self.html_result_dir = dir_paths['result']
self._jsons = jsons
self._path = path
self.yang_models = dir_paths['yang_models']
self._parsed_yang = yangParser.parse(self._path)
self.implementations: t.List[Implementation] = []
self._parse_all(name, git_commit_hash, yang_modules, schema_base,
dir_paths['save'], aditional_info, submodule_name)
del self._jsons
def __resolve_submodule_case(self, field):
if self.module_type == 'submodule':
LOGGER.debug(
'Getting parent information because file {} is a submodule'.format(
self.__path))
yang_file = self.__find_file(self.belongs_to)
if yang_file is None:
return None
parsed_parent_yang = yangParser.parse(os.path.abspath(yang_file))
try:
return parsed_parent_yang.search(field)[0].arg
except:
if field == 'prefix':
return None
else:
return MISSING_ELEMENT
else:
try:
return self.__parsed_yang.search(field)[0].arg
except:
if field == 'prefix':
return None
else:
return MISSING_ELEMENT
def resolve_organization(path):
"""Parse yang file and resolve organization out of the module. If the module
is a submodule find it's parent and resolve its organization
Arguments:
:param path: (str) path to a file to parse and resolve a organization
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
organization = ''
try:
namespace = yangParser.parse(os.path.abspath(path)) \
.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(path)) \
.search('belongs-to')[0].arg
except:
break
try:
yang_file = find_first_file('/'.join(path.split('/')[:-1]),
belongs_to + '.yang',
belongs_to + '@*.yang')
namespace = yangParser.parse(
os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
try:
yang_file = find_first_file('/'.join(path.split('/')[:-2]),
belongs_to + '.yang',
belongs_to + '@*.yang')
namespace = yangParser.parse(
os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(
':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
organization = MISSING_ELEMENT
return organization
import os
import json
import filecmp
from utility import yangParser
conflicting = []
fnames = {}
top = '/backend/tests/resources/yangmodels/yang/'
for dirname, _, files in os.walk(top):
dirname = os.path.join(top, dirname)
for f in files:
if f.endswith('.yang'):
parsed = yangParser.parse(os.path.join(dirname, f))
try:
revision = parsed.search('revision')[0].arg
except:
revision = '1970-01-01'
if f not in fnames:
fnames[f] = {}
if not revision in fnames[f]:
fnames[f][revision] = os.path.join(dirname, f)
else:
if not filecmp.cmp(fnames[f][revision],
os.path.join(dirname, f),
shallow=False):
conflicting.append(
(fnames[f][revision], os.path.join(dirname, f)))
with open('conflicting.json', 'w') as f:
json.dump(conflicting, f)
def get_total_and_passed(dir: str) -> t.Tuple[int, int]:
"""Get the number of yang files in a specified directory and the
number that passed compilation.
Arguments:
:param path_dir: (str) path to the directory where to search for
yang files
:return: tuple containing the number of yang files and the number
that passed compilation respectively
"""
passed = 0
num_in_catalog = 0
yang_modules = list_yang_modules_recursive(dir)
num_of_modules = len(yang_modules)
checked = {}
for i, module_path in enumerate(yang_modules, start=1):
filename = os.path.basename(module_path)
LOGGER.debug('{} out of {}: {}'.format(i, num_of_modules, filename))
if filename in checked.keys():
passed += checked[filename]['passed']
num_in_catalog += checked[filename]['in-catalog']
continue
checked[filename] = {}
checked[filename]['passed'] = False
checked[filename]['in-catalog'] = False
revision = None
try:
parsed_yang = yangParser.parse(os.path.abspath(module_path))
except yangParser.ParseException:
continue
results = parsed_yang.search('revision')
if results:
revision = results[0].arg
organization = resolve_organization(module_path, parsed_yang)
name = filename.split('.')[0].split('@')[0]
if revision is None:
revision = '1970-01-01'
organization = organization.replace(' ', '%20')
if ',' in organization:
path = os.path.join(yangcatalog_api_prefix, 'search/name', name)
module_exists = requests.get(path, headers=json_headers)
if module_exists.status_code % 100 == 2:
data = module_exists.json()
org = data['yang-catalog:modules']['module'][0]['organization']
rev = data['yang-catalog:modules']['module'][0]['revision']
status = data['yang-catalog:modules']['module'][0].get(
'compilation-status')
if org == organization and rev == revision:
if 'passed' == status:
passed += 1
num_in_catalog += 1
else:
LOGGER.error('Could not make request on path {}'.format(path))
else:
mod = '{}@{}_{}'.format(name, revision, organization)
data = all_modules_data_unique.get(mod)
if data is not None:
if 'passed' == data.get('compilation-status'):
passed += 1
checked[filename]['passed'] = True
checked[filename]['in-catalog'] = True
num_in_catalog += 1
else:
LOGGER.error('module {} does not exist'.format(mod))
return num_in_catalog, passed
def resolve_organization(path, parsed_yang):
"""Parse yang file and resolve organization out of the module. If the module
is a submodule find it's parent and resolve its organization
Arguments:
:param path: (str) path to a file to parse and resolve a organization
:param parsed_yang (object) pyang parsed yang file object
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
organization = ''
try:
temp_organization = parsed_yang.search('organization')[0].arg.lower()
if 'cisco' in temp_organization or 'CISCO' in temp_organization:
organization = 'cisco'
elif 'ietf' in temp_organization or 'IETF' in temp_organization:
organization = 'ietf'
except:
pass
try:
namespace = parsed_yang.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'cisco' in namespace or 'CISCO' in namespace:
organization = 'cisco'
elif 'ietf' in namespace or 'IETF' in namespace:
organization = 'ietf'
elif 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
except:
try:
belongs_to = parsed_yang.search('belongs-to')[0].arg
except:
organization = MISSING_ELEMENT
return organization
try:
yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'cisco' in namespace or 'CISCO' in namespace:
organization = 'cisco'
elif 'ietf' in namespace or 'IETF' in namespace:
organization = 'ietf'
elif 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
return organization
except:
try:
yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'cisco' in namespace or 'CISCO' in namespace:
organization = 'cisco'
elif 'ietf' in namespace or 'IETF' in namespace:
organization = 'ietf'
elif 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
except:
organization = MISSING_ELEMENT
return organization
def __init__(self, yang_models_dir, log_directory, path, html_result_dir, jsons, temp_dir,
is_vendor=False, is_yang_lib=False, data=None,
is_vendor_imp_inc=False, run_integrity=False):
"""
Preset Modules class to parse yang module and save data to it.
:param yang_models_dir: (str) directory with all yang modules from
github https://github.com/YangModels/yang
:param log_directory: (str) directory where the log file is saved
:param path: (str) path to yang file being parsed
:param html_result_dir: (str) path to directory with html result
files
:param jsons: (obj) LoadFiles class containing all the json
and html files with parsed results
:param temp_dir: (str) path to temporary directory
:param is_vendor: (boolean) if we parsing vendor files (cisco, huawei, ..)
or sdo files (ietf, ieee, ...)
:param is_yang_lib: (boolean) if we are parsing file from yang_lib
capability file
:param data: (dict) data from yang_lib capability file with additional
information
:param is_vendor_imp_inc: (boolean) Obsolete
:param run_integrity (boolean) if we are running integrity as well. If true
part of the data parsed are not needed and therefor not
parsed
"""
global LOGGER
LOGGER = log.get_logger('modules', log_directory + '/parseAndPopulate.log')
self.run_integrity = run_integrity
self.__temp_dir = temp_dir
self.__missing_submodules = []
self.__missing_modules = []
self.__missing_namespace = None
self.__missing_revision = None
self.is_yang_lib = is_yang_lib
self.html_result_dir = html_result_dir
self.jsons = jsons
self.__is_vendor = is_vendor
self.revision = '*'
self.__path = path
self.features = []
self.deviations = []
self.yang_models = yang_models_dir
if is_vendor:
if is_yang_lib:
self.deviations = data['deviations']
self.features = data['features']
self.revision = data['revision']
if self.revision is None:
self.revision = '*'
self.__path = self.__find_file(data['name'], self.revision)
else:
self.features = self.\
__resolve_deviations_and_features('features=', data)
self.deviations = \
self.__resolve_deviations_and_features('deviations=', data)
if 'revision' in data:
revision_and_more = data.split('revision=')[1]
revision = revision_and_more.split('&')[0]
self.revision = revision
self.__path = self.__find_file(data.split('&')[0], self.revision)
else:
self.__path = path
if is_vendor_imp_inc:
self.__is_vendor = True
if self.__path:
self.name = None
self.organization = None
self.ietf_wg = None
self.namespace = None
self.schema = None
self.generated_from = None
self.maturity_level = None
self.document_name = None
self.author_email = None
self.reference = None
self.tree = None
self.expired = None
self.expiration_date = None
self.module_classification = None
self.compilation_status = None
self.compilation_result = {}
self.prefix = None
self.yang_version = None
self.description = None
self.contact = None
self.belongs_to = None
self.submodule = []
self.dependencies = []
self.module_type = None
self.tree_type = None
self.semver = None
self.derived_semver = None
self.implementation = []
self.imports = []
self.json_submodules = json.dumps([])
self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path))
if self.__parsed_yang is None:
raise ParseException(path)
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path.split('&')[0])
def __init__(self,
yang_models_dir,
log_directory,
path,
html_result_dir,
jsons,
temp_dir,
is_vendor=False,
is_yang_lib=False,
data=None,
is_vendor_imp_inc=False,
run_integrity=False):
global LOGGER
LOGGER = log.get_logger('modules',
log_directory + '/parseAndPopulate.log')
self.run_integrity = run_integrity
self.__temp_dir = temp_dir
self.__missing_submodules = []
self.__missing_modules = []
self.__missing_namespace = None
self.__missing_revision = None
self.is_yang_lib = is_yang_lib
self.html_result_dir = html_result_dir
self.jsons = jsons
self.__is_vendor = is_vendor
self.revision = '*'
self.__path = path
self.features = []
self.deviations = []
self.yang_models = yang_models_dir
if is_vendor:
if is_yang_lib:
self.deviations = data['deviations']
self.features = data['features']
self.revision = data['revision']
if self.revision is None:
self.revision = '*'
self.__path = self.__find_file(data['name'], self.revision)
else:
self.features = self.\
__resolve_deviations_and_features('features=', data)
self.deviations = \
self.__resolve_deviations_and_features('deviations=', data)
if 'revision' in data:
revision_and_more = data.split('revision=')[1]
revision = revision_and_more.split('&')[0]
self.revision = revision
self.__path = self.__find_file(
data.split('&')[0], self.revision)
else:
self.__path = path
if is_vendor_imp_inc:
self.__is_vendor = True
if self.__path:
self.name = None
self.organization = None
self.ietf_wg = None
self.namespace = None
self.schema = None
self.generated_from = None
self.maturity_level = None
self.document_name = None
self.author_email = None
self.reference = None
self.tree = None
self.expired = None
self.expiration_date = None
self.module_classification = None
self.compilation_status = None
self.compilation_result = {}
self.prefix = None
self.yang_version = None
self.description = None
self.contact = None
self.belongs_to = None
self.submodule = []
self.dependencies = []
self.module_type = None
self.tree_type = None
self.semver = None
self.derived_semver = None
self.implementation = []
self.imports = []
self.json_submodules = json.dumps([])
self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path))
if self.__parsed_yang is None:
raise ParseException(path)
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
path.split('&')[0])
def add_modules():
"""Endpoint is used to add new modules using the API.
PUT request is used for updating each module in request body.
POST request is used for creating new modules that are not in ConfD/Redis yet.
First it checks if the sent request is ok and if so, it will send a another request
to the receiver which will work on adding/updating modules while this request
will send a "job_id" in the response back to the user.
User is able to check success of the job using this "job_id"
the job process.
:return response with "job_id" that user can use to check whether
the job is still running or Failed or Finished successfully.
"""
if not request.json:
abort(400,
description=
'bad request - you need to input json body that conforms with'
' module-metadata.yang module. Received no json')
body = request.json
modules_cont = body.get('modules')
if modules_cont is None:
abort(
400,
description=
'bad request - "modules" json object is missing and is mandatory')
module_list = modules_cont.get('module')
if module_list is None:
abort(400,
description=
'bad request - "module" json list is missing and is mandatory')
app.logger.info('Adding modules with body\n{}'.format(
json.dumps(body, indent=2)))
dst_path = os.path.join(
ac.d_save_requests,
'sdo-{}.json'.format(datetime.utcnow().strftime(backup_date_format)))
if not os.path.exists(ac.d_save_requests):
os.mkdir(ac.d_save_requests)
with open(dst_path, 'w') as f:
json.dump(body, f)
response = app.confdService.put_module_metadata(json.dumps(body))
if response.status_code != 200 and response.status_code != 201 and response.status_code != 204:
abort(
400,
description=
'The body you have provided could not be parsed. ConfD error text:\n{}\n'
'Error code: {}'.format(response.text, response.status_code))
direc_num = 0
while os.path.isdir(os.path.join(ac.d_temp, str(direc_num))):
direc_num += 1
direc = os.path.join(ac.d_temp, str(direc_num))
try:
os.makedirs(direc)
except OSError as e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
repos: t.Dict[str, repoutil.RepoUtil] = {}
warning = []
missing_msg = 'bad request - at least one module object is missing mandatory field {}'
for module in module_list:
app.logger.debug(module)
source_file: dict = module.get('source-file')
if source_file is None:
abort(400, description=missing_msg.format('source-file'))
organization_sent = module.get('organization')
if organization_sent is None:
abort(400, description=missing_msg.format('organization'))
name = module.get('name')
if name is None:
abort(400, description=missing_msg.format('name'))
revision = module.get('revision')
if revision is None:
abort(400, description=missing_msg.format('revision'))
if request.method == 'POST':
# Check if the module is already in Redis
redis_module = get_mod_redis(module)
if redis_module != {}:
continue
module_path = source_file.get('path')
if module_path is None:
abort(400, description=missing_msg.format('source-file["path"]'))
repo_name = source_file.get('repository')
if repo_name is None:
abort(400,
description=missing_msg.format('source-file["repository"]'))
owner = source_file.get('owner')
if owner is None:
abort(400, description=missing_msg.format('source-file["owner"]'))
dir_in_repo = os.path.dirname(module_path)
repo_url = os.path.join(github_url, owner, repo_name)
if repo_url not in repos:
repos[repo_url] = get_repo(repo_url, owner, repo_name)
# needed to later construct the schema
source_file['commit-hash'] = repos[repo_url].get_commit_hash(
branch=source_file.get('branch', 'HEAD'))
save_to = os.path.join(direc, owner,
repo_name.split('.')[0], dir_in_repo)
try:
os.makedirs(save_to)
except OSError as e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
try:
shutil.copy(os.path.join(repos[repo_url].local_dir, module_path),
save_to)
except FileNotFoundError:
app.logger.exception('Problem with file {}'.format(module_path))
warning.append('{} does not exist'.format(
os.path.join(repo_url, 'blob', source_file['commit-hash'],
module_path)))
continue
organization_parsed = ''
try:
namespace = yangParser.parse(os.path.abspath('{}/{}'.format(save_to, os.path.basename(module_path)))) \
.search('namespace')[0].arg
organization_parsed = organization_by_namespace(namespace)
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(os.path.join(repos[repo_url].local_dir, module_path))) \
.search('belongs-to')[0].arg
except:
break
namespace = yangParser.parse(
os.path.abspath('{}/{}/{}.yang'.format(
repos[repo_url].local_dir,
os.path.dirname(module_path),
belongs_to))).search('namespace')[0].arg
organization_parsed = organization_by_namespace(namespace)
break
resolved_authorization = authorize_for_sdos(request, organization_sent,
organization_parsed)
if not resolved_authorization:
shutil.rmtree(direc)
abort(401, description='Unauthorized for server unknown reason')
if 'organization' in repr(resolved_authorization):
warning.append('{} {}'.format(os.path.basename(module_path),
resolved_authorization))
with open(os.path.join(direc, 'request-data.json'), 'w') as f:
json.dump(body, f)
arguments = [
'POPULATE-MODULES', '--sdo', '--dir', direc, '--api', '--credentials',
ac.s_confd_credentials[0], ac.s_confd_credentials[1]
]
job_id = ac.sender.send('#'.join(arguments))
app.logger.info('Running populate.py with job_id {}'.format(job_id))
if len(warning) > 0:
return {
'info': 'Verification successful',
'job-id': job_id,
'warnings': [{
'warning': val
} for val in warning]
}
else:
return ({'info': 'Verification successful', 'job-id': job_id}, 202)
