def test_check_depencendies(self): good = parse(self.module_path('good')) assert good all_imports, missing_imports = itg.check_dependencies('import', good, self.module_dir, self.yang_models) self.assertSetEqual(all_imports, {'invalid-revision'}) self.assertFalse(missing_imports) all_includes, missing_includes = itg.check_dependencies('include', good, self.module_dir, self.yang_models) self.assertSetEqual(all_includes, {'invalid-namespace', 'l1-dependency'}) self.assertFalse(missing_includes) missing_import = parse(self.module_path('missing-import')) assert missing_import all_imports, missing_imports = itg.check_dependencies('import', missing_import, self.module_dir, self.yang_models) self.assertSetEqual(all_imports, {'nonexistent'}) self.assertSetEqual(missing_imports, {'nonexistent'}) missing_include = parse(self.module_path('missing-include')) assert missing_include all_includes, missing_includes = itg.check_dependencies('include', missing_include, self.module_dir, self.yang_models) self.assertSetEqual(all_includes, {'nonexistent'}) self.assertSetEqual(missing_includes, {'nonexistent'})
def test_check_revision(self): good = parse(self.module_path('good')) assert good self.assertTrue(itg.check_revision(good)) missing_revision = parse(self.module_path('missing-revision')) assert missing_revision self.assertFalse(itg.check_revision(missing_revision)) invalid_revision = parse(self.module_path('invalid-revision')) assert invalid_revision self.assertFalse(itg.check_revision(invalid_revision))
def test_check_namespace(self): good = parse(self.module_path('good')) assert good self.assertTrue(itg.check_namespace(good)) missing_namespace = parse(self.module_path('missing-namespace')) assert missing_namespace self.assertFalse(itg.check_namespace(missing_namespace)) invalid_namespace = parse(self.module_path('invalid-namespace')) assert invalid_namespace self.assertFalse(itg.check_namespace(invalid_namespace))
def get_available_commit_hash(module: dict, commit_hash_list: list): name = module.get('name') schema = module.get('schema', '') revision = module.get('revision') branch = get_branch_from_schema(schema) available_commit_hash = '' for commit_hash in commit_hash_list: new_schema = schema.replace(branch, commit_hash) if new_schema not in requests_done: requests_done.append(new_schema) response = requests.get(new_schema) if response.status_code == 200: module_text = response.text try: module_revision = yangParser.parse(module_text).search( 'revision')[0].arg except: module_revision = '1970-01-01' key = '{}@{}'.format(name, module_revision) available_schemas[key] = new_schema if revision == module_revision: available_commit_hash = commit_hash break return available_commit_hash
def find_first_file(directory, pattern, pattern_with_revision): """Find first yang file based on name or name and revision :param directory: (str) directory where to look for a file :param pattern: (str) name of the yang file :param pattern_with_revision: name and revision of the in format <name>@<revision> :return path to current directory """ for root, dirs, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern_with_revision): filename = os.path.join(root, basename) return filename for root, dirs, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern): filename = os.path.join(root, basename) try: revision = yangParser.parse(filename).search('revision')[0]\ .arg except: revision = '1970-01-01' if '*' not in pattern_with_revision: if revision in pattern_with_revision: return filename else: return filename
def get_specifics(path_dir): """Get amount of yang files in specified directory and the amount that passed compilation Arguments: :param path_dir: (str) path to directory where we are searching for yang files :return: list containing amount of yang files and amount that pass compilation respectively """ passed = 0 num_in_catalog = 0 yang_modules = list_of_yang_modules_in_subdir(path_dir) yang_modules_length = len(yang_modules) x = 0 for mod_git in yang_modules: x += 1 LOGGER.info("{} out of {} getting specifics from {}".format(x, yang_modules_length, path_dir)) try: parsed_yang = yangParser.parse(os.path.abspath(mod_git)) revision = parsed_yang.search('revision')[0].arg except: continue organization = resolve_organization(mod_git, parsed_yang) name = mod_git.split('/')[-1].split('.')[0].split('@')[0] if revision is None: revision = '1970-01-01' if name is None or organization is None: continue if ',' in organization: organization = organization.replace(' ', '%20') path = '{}search/name/{}'.format(yangcatalog_api_prefix, name) module_exist = requests.get(path, auth=(auth[0], auth[1]), headers={'Accept': 'application/vnd.yang.data+json', 'Content-Type': 'application/vnd.yang.data+json'}) if repr(module_exist.status_code).startswith('20'): data = module_exist.json() org = data['yang-catalog:modules']['module'][0]['organization'] rev = data['yang-catalog:modules']['module'][0]['revision'] status = data['yang-catalog:modules']['module'][0].get('compilation-status') if org == organization and rev == revision: if 'passed' == status: passed += 1 num_in_catalog += 1 else: LOGGER.error('Could not send request with path {}'.format(path)) else: organization = organization.replace(' ', '%20') mod = '{}@{}_{}'.format(name, revision, organization) data = all_modules_data_unique.get(mod) if data is not None: if 'passed' == data.get('compilation-status'): passed += 1 num_in_catalog += 1 else: LOGGER.error('module {} does not exist'.format(mod)) return [num_in_catalog, passed]
def resolve_revision(yang_file): """ search for revision in file "yang_file" :param yang_file: yang file :return: revision of the yang file """ try: parsed_yang = yangParser.parse(os.path.abspath(yang_file)) revision = parsed_yang.search('revision')[0].arg except: revision = '1970-01-01' return revision
def __init__(self, name: str, path: str, jsons: LoadFiles, dir_paths: DirPaths, git_commit_hash: str, yang_modules: dict, schema_base: str, aditional_info: t.Optional[t.Dict[str, str]] = None, submodule_name: t.Optional[str] = None, data: t.Optional[t.Union[str, dict]] = None): real_path = path # these are required for self._find_file() to work self.yang_models = dir_paths['yang_models'] self._path = path self.features = [] self.deviations = [] if isinstance(data, str): # string from a capabilities file self.features = self._resolve_deviations_and_features( 'features=', data) deviation_names = self._resolve_deviations_and_features( 'deviations=', data) for name in deviation_names: deviation = {'name': name} yang_file = self._find_file(name) if yang_file is None: deviation['revision'] = '1970-01-01' else: try: deviation['revision'] = yangParser.parse(os.path.abspath(yang_file)) \ .search('revision')[0].arg except: deviation['revision'] = '1970-01-01' self.deviations.append(deviation) if 'revision' in data: revision_and_more = data.split('revision=')[1] revision = revision_and_more.split('&')[0] self.revision = revision real_path = self._find_file(data.split('&')[0], self.revision) elif isinstance(data, dict): # dict parsed out from a ietf-yang-library file self.deviations = data['deviations'] self.features = data['features'] self.revision = data['revision'] real_path = self._find_file(data['name'], self.revision) if not real_path: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) super().__init__(name, os.path.abspath(real_path), jsons, dir_paths, git_commit_hash, yang_modules, schema_base, aditional_info, submodule_name)
def resolve_revision(yang_file: str): """ Search for revision in file "yang_file" Argument: :param yang_file (str) full path to the yang file :return: revision of the yang file """ try: parsed_yang = yangParser.parse(os.path.abspath(yang_file)) revision = parsed_yang.search('revision')[0].arg except Exception: revision = '1970-01-01' return revision
def get_latest_revision(f, LOGGER): """ Search for revision in yang file :param f: yang file :return: revision of the file "f" """ stmt = yangParser.parse(f) if stmt is None: # In case of invalid YANG syntax, None is returned LOGGER.info('Cannot yangParser.parse ' + f) return None rev = stmt.search_one('revision') if rev is None: return None return rev.arg
def __resolve_submodule(self): try: submodules = self.__parsed_yang.search('include') except: return if len(submodules) == 0: return for chunk in submodules: dep = self.Dependencies() sub = self.Submodules() sub.name = chunk.arg if len(chunk.search('revision-date')) > 0: sub.revision = chunk.search('revision-date')[0].arg if sub.revision: yang_file = self.__find_file(sub.name, sub.revision, True) dep.revision = sub.revision else: yang_file = self.__find_file(sub.name, submodule=True) try: sub.revision = \ yangParser.parse(os.path.abspath(yang_file)).search( 'revision')[0].arg except: sub.revision = '1970-01-01' if yang_file is None: LOGGER.error('Module can not be found') continue path = '/'.join(self.schema.split('/')[0:-1]) path += '/{}'.format(yang_file.split('/')[-1]) if yang_file: sub.schema = path dep.name = sub.name dep.schema = sub.schema self.dependencies.append(dep) self.submodule.append(sub) self.json_submodules = json.dumps([{ 'name': self.submodule[x].name, 'schema': self.submodule[x].schema, 'revision': self.submodule[x].revision } for x in range(0, len(self.submodule))])
def find_first_file(directory: str, pattern: str, pattern_with_revision: str, yang_models_dir: str = '') -> t.Optional[str]: """ Search for the first file in 'directory' which either match 'pattern' or 'pattern_with_revision' string. Arguments: :param directory (str) directory where to look for a file :param pattern (str) name of the yang file :param pattern_with_revision (str) name and revision of the module in format <name>@<revision> :param yang_models_dir (str) path to the directory where YangModels/yang repo is cloned :return (str) path to matched file """ def match_file(directory, pattern) -> t.Optional[str]: for root, _, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern): return os.path.join(root, basename) rfcs_dir = '{}/standard/ietf/RFC'.format(yang_models_dir) standards_dir = '{}/standard'.format(yang_models_dir) experimental_dir = '{}/experimental/ietf-extracted-YANG-modules'.format(yang_models_dir) paths_to_check = [directory, rfcs_dir, standards_dir, experimental_dir, yang_models_dir] patterns_order = [] if '*' not in pattern_with_revision: patterns_order = [pattern_with_revision, pattern] else: patterns_order = [pattern, pattern_with_revision] for path in paths_to_check: for pattern in patterns_order: filename = match_file(path, pattern) if filename: try: parsed = yangParser.parse(filename) except: continue results = parsed.search('revision') if results: revision = results[0].arg else: revision = '1970-01-01' if '*' not in pattern_with_revision: if revision in pattern_with_revision: return filename else: return filename
def _resolve_submodule(self, dependencies: t.List[Dependency], submodule: t.List[Submodules]): LOGGER.debug('Resolving submodules') try: submodules = self._parsed_yang.search('include') except: return if len(submodules) == 0: return for chunk in submodules: dep = Dependency() sub = Submodules() sub.name = chunk.arg if len(chunk.search('revision-date')) > 0: sub.revision = chunk.search('revision-date')[0].arg if sub.revision: yang_file = self._find_file(sub.name, sub.revision) dep.revision = sub.revision else: yang_file = self._find_file(sub.name) if yang_file: try: sub.revision = \ yangParser.parse(os.path.abspath(yang_file)).search('revision')[0].arg except: sub.revision = '1970-01-01' else: sub.revision = '1970-01-01' if yang_file is None: LOGGER.error('Module can not be found') continue if self.schema: sub_schema = '/'.join( (*self.schema.split('/')[0:-1], yang_file.split('/')[-1])) else: sub_schema = None if yang_file: sub.schema = sub_schema dep.name = sub.name dep.schema = sub.schema dependencies.append(dep) submodule.append(sub)
def resolve_organization(path: str, parsed_yang) -> str: """Parse yang file and resolve organization out of the module. If the module is a submodule find it's parent and resolve its organization Arguments: :param path: (str) path to a file to parse and resolve a organization :param parsed_yang (object) pyang parsed yang file object :return: (str) organization the yang file belongs to """ organization = None results = parsed_yang.search('organization') if results: result = results[0].arg.lower() if 'cisco' in result: organization = 'cisco' elif 'ietf' in result: organization = 'ietf' namespace = None results = parsed_yang.search('namespace') if results: namespace = results[0].arg.lower() else: results = parsed_yang.search('belongs-to') if results: belongs_to = results[0].arg pattern = '{}.yang'.format(belongs_to) pattern_with_revision = '{}@*.yang'.format(belongs_to) results = None yang_file = find_first_file(os.path.dirname(path), pattern, pattern_with_revision) if yang_file is None: yang_file = find_first_file('/'.join(path.split('/')[:-2]), pattern, pattern_with_revision) if yang_file is not None: try: parsed = yangParser.parse(os.path.abspath(yang_file)) if parsed: results = parsed.search('namespace') if results: namespace = results[0].arg.lower() except yangParser.ParseException: pass if namespace is None: return MISSING_ELEMENT else: return match_organization(namespace, organization)
def get_latest_revision(path: str, LOGGER: logging.Logger): """ Search for the latest revision in yang file Arguments: :param path (str) full path to the yang file :param LOGGER (logging.Logger) formated logger with the specified name :return revision of the module at the given path """ try: stmt = yangParser.parse(path) result = stmt.search_one('revision') assert result rev = result.arg except Exception: LOGGER.error('Cannot yangParser.parse {}'.format(path)) rev = None # In case of invalid YANG syntax, None is returned return rev
def add_vendor_information(self, vendor, platform_data, software_version, os_version, feature_set, os_type, confarmance_type, capability, netconf_version, integrity_checker, split): for data in platform_data: implementation = self.Implementations() implementation.vendor = vendor implementation.platform = data['platform'] implementation.software_version = software_version implementation.software_flavor = data['software-flavor'] implementation.os_version = os_version implementation.feature_set = feature_set implementation.os_type = os_type implementation.feature = self.features implementation.capability = capability implementation.netconf_version = netconf_version if self.is_yang_lib: for deviation in self.deviations: devs = implementation.Deviations() devs.name = deviation['name'] devs.revision = deviation['revision'] implementation.deviations.append(devs) else: for name in self.deviations: devs = implementation.Deviations() devs.name = name yang_file = self.__find_file(name) if yang_file is None: devs.revision = '1970-01-01' else: try: s = yang_file.split('/') key = '/'.join(split[0:-1]) integrity_checker.remove_one(key, s[-1]) devs.revision = yangParser.parse(os.path.abspath(yang_file)) \ .search('revision')[0].arg except: devs.revision = '1970-01-01' implementation.deviations.append(devs) implementation.conformance_type = confarmance_type self.implementation.append(implementation)
def check(path: str, directory: str, yang_models_dir: str, sdo: bool): try: parsed_module = yangParser.parse(path) except yangParser.ParseException: return if not check_revision(parsed_module): missing_revisions.add(path) if not check_namespace(parsed_module): missing_namespaces.add(path) all_imports, broken_imports = check_dependencies('import', parsed_module, directory, yang_models_dir) if broken_imports: missing_modules[path] |= broken_imports all_includes, broken_includes = check_dependencies('include', parsed_module, directory, yang_models_dir) if broken_includes: missing_submodules[path] |= broken_includes if not sdo: all_dependencies = all_imports | all_includes for module in all_dependencies: if module in unused_modules[directory]: unused_modules[directory].remove(module) modules_to_check.append(module)
def _resolve_submodule_case(self, field) -> t.Optional[str]: if self.module_type == 'submodule': LOGGER.debug( 'Getting parent information because file {} is a submodule'. format(self._path)) if self.belongs_to: yang_file = self._find_file(self.belongs_to) else: return None if yang_file is None: return None try: parsed_parent_yang = yangParser.parse( os.path.abspath(yang_file)) return parsed_parent_yang.search(field)[0].arg except: return None else: try: return self._parsed_yang.search(field)[0].arg except: return None
def __init__(self, name: str, path: str, jsons: LoadFiles, dir_paths: DirPaths, git_commit_hash: str, yang_modules: dict, schema_base: str, aditional_info: t.Optional[t.Dict[str, str]], submodule_name: t.Optional[str]): """ Initialize and parse everything out of a module. Arguments: :param name: (str) name of the module (not parsed out of the module) :param path: (str) path to yang file being parsed :param jsons: (obj) LoadFiles class containing all the json and html files with parsed results :param dir_paths: (dict) paths to various needed directories according to configuration :param git_commit_hash: (str) name of the git commit hash where we can find the module :param yang_modules: (dict) yang modules we've already parsed :param schema_base: (str) url to a raw module on github up to and not including the path of the file in the repo :param aditional_info: (dict) some aditional information about module given from client :param submodule_name: (str) name of the git submodule the yang module belongs to """ global LOGGER LOGGER = log.get_logger( 'modules', '{}/parseAndPopulate.log'.format(dir_paths['log'])) config = create_config() self._web_uri = config.get('Web-Section', 'my-uri', fallback='https://yangcatalog.org') self.html_result_dir = dir_paths['result'] self._jsons = jsons self._path = path self.yang_models = dir_paths['yang_models'] self._parsed_yang = yangParser.parse(self._path) self.implementations: t.List[Implementation] = [] self._parse_all(name, git_commit_hash, yang_modules, schema_base, dir_paths['save'], aditional_info, submodule_name) del self._jsons
def __resolve_submodule_case(self, field): if self.module_type == 'submodule': LOGGER.debug( 'Getting parent information because file {} is a submodule'.format( self.__path)) yang_file = self.__find_file(self.belongs_to) if yang_file is None: return None parsed_parent_yang = yangParser.parse(os.path.abspath(yang_file)) try: return parsed_parent_yang.search(field)[0].arg except: if field == 'prefix': return None else: return MISSING_ELEMENT else: try: return self.__parsed_yang.search(field)[0].arg except: if field == 'prefix': return None else: return MISSING_ELEMENT
def resolve_organization(path): """Parse yang file and resolve organization out of the module. If the module is a submodule find it's parent and resolve its organization Arguments: :param path: (str) path to a file to parse and resolve a organization :return: list containing amount of yang files and amount that pass compilation respectively """ organization = '' try: namespace = yangParser.parse(os.path.abspath(path)) \ .search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT except: while True: try: belongs_to = yangParser.parse(os.path.abspath(path)) \ .search('belongs-to')[0].arg except: break try: yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang', belongs_to + '@*.yang') namespace = yangParser.parse( os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT break except: try: yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang', belongs_to + '@*.yang') namespace = yangParser.parse( os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split( ':')[0] if organization == '': organization = MISSING_ELEMENT break except: organization = MISSING_ELEMENT return organization
import os import json import filecmp from utility import yangParser conflicting = [] fnames = {} top = '/backend/tests/resources/yangmodels/yang/' for dirname, _, files in os.walk(top): dirname = os.path.join(top, dirname) for f in files: if f.endswith('.yang'): parsed = yangParser.parse(os.path.join(dirname, f)) try: revision = parsed.search('revision')[0].arg except: revision = '1970-01-01' if f not in fnames: fnames[f] = {} if not revision in fnames[f]: fnames[f][revision] = os.path.join(dirname, f) else: if not filecmp.cmp(fnames[f][revision], os.path.join(dirname, f), shallow=False): conflicting.append( (fnames[f][revision], os.path.join(dirname, f))) with open('conflicting.json', 'w') as f: json.dump(conflicting, f)
def get_total_and_passed(dir: str) -> t.Tuple[int, int]: """Get the number of yang files in a specified directory and the number that passed compilation. Arguments: :param path_dir: (str) path to the directory where to search for yang files :return: tuple containing the number of yang files and the number that passed compilation respectively """ passed = 0 num_in_catalog = 0 yang_modules = list_yang_modules_recursive(dir) num_of_modules = len(yang_modules) checked = {} for i, module_path in enumerate(yang_modules, start=1): filename = os.path.basename(module_path) LOGGER.debug('{} out of {}: {}'.format(i, num_of_modules, filename)) if filename in checked.keys(): passed += checked[filename]['passed'] num_in_catalog += checked[filename]['in-catalog'] continue checked[filename] = {} checked[filename]['passed'] = False checked[filename]['in-catalog'] = False revision = None try: parsed_yang = yangParser.parse(os.path.abspath(module_path)) except yangParser.ParseException: continue results = parsed_yang.search('revision') if results: revision = results[0].arg organization = resolve_organization(module_path, parsed_yang) name = filename.split('.')[0].split('@')[0] if revision is None: revision = '1970-01-01' organization = organization.replace(' ', '%20') if ',' in organization: path = os.path.join(yangcatalog_api_prefix, 'search/name', name) module_exists = requests.get(path, headers=json_headers) if module_exists.status_code % 100 == 2: data = module_exists.json() org = data['yang-catalog:modules']['module'][0]['organization'] rev = data['yang-catalog:modules']['module'][0]['revision'] status = data['yang-catalog:modules']['module'][0].get( 'compilation-status') if org == organization and rev == revision: if 'passed' == status: passed += 1 num_in_catalog += 1 else: LOGGER.error('Could not make request on path {}'.format(path)) else: mod = '{}@{}_{}'.format(name, revision, organization) data = all_modules_data_unique.get(mod) if data is not None: if 'passed' == data.get('compilation-status'): passed += 1 checked[filename]['passed'] = True checked[filename]['in-catalog'] = True num_in_catalog += 1 else: LOGGER.error('module {} does not exist'.format(mod)) return num_in_catalog, passed
def resolve_organization(path, parsed_yang): """Parse yang file and resolve organization out of the module. If the module is a submodule find it's parent and resolve its organization Arguments: :param path: (str) path to a file to parse and resolve a organization :param parsed_yang (object) pyang parsed yang file object :return: list containing amount of yang files and amount that pass compilation respectively """ organization = '' try: temp_organization = parsed_yang.search('organization')[0].arg.lower() if 'cisco' in temp_organization or 'CISCO' in temp_organization: organization = 'cisco' elif 'ietf' in temp_organization or 'IETF' in temp_organization: organization = 'ietf' except: pass try: namespace = parsed_yang.search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'cisco' in namespace or 'CISCO' in namespace: organization = 'cisco' elif 'ietf' in namespace or 'IETF' in namespace: organization = 'ietf' elif 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT except: try: belongs_to = parsed_yang.search('belongs-to')[0].arg except: organization = MISSING_ELEMENT return organization try: yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang' , belongs_to + '@*.yang') namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'cisco' in namespace or 'CISCO' in namespace: organization = 'cisco' elif 'ietf' in namespace or 'IETF' in namespace: organization = 'ietf' elif 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT return organization except: try: yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang' , belongs_to + '@*.yang') namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'cisco' in namespace or 'CISCO' in namespace: organization = 'cisco' elif 'ietf' in namespace or 'IETF' in namespace: organization = 'ietf' elif 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT except: organization = MISSING_ELEMENT return organization
def __init__(self, yang_models_dir, log_directory, path, html_result_dir, jsons, temp_dir, is_vendor=False, is_yang_lib=False, data=None, is_vendor_imp_inc=False, run_integrity=False): """ Preset Modules class to parse yang module and save data to it. :param yang_models_dir: (str) directory with all yang modules from github https://github.com/YangModels/yang :param log_directory: (str) directory where the log file is saved :param path: (str) path to yang file being parsed :param html_result_dir: (str) path to directory with html result files :param jsons: (obj) LoadFiles class containing all the json and html files with parsed results :param temp_dir: (str) path to temporary directory :param is_vendor: (boolean) if we parsing vendor files (cisco, huawei, ..) or sdo files (ietf, ieee, ...) :param is_yang_lib: (boolean) if we are parsing file from yang_lib capability file :param data: (dict) data from yang_lib capability file with additional information :param is_vendor_imp_inc: (boolean) Obsolete :param run_integrity (boolean) if we are running integrity as well. If true part of the data parsed are not needed and therefor not parsed """ global LOGGER LOGGER = log.get_logger('modules', log_directory + '/parseAndPopulate.log') self.run_integrity = run_integrity self.__temp_dir = temp_dir self.__missing_submodules = [] self.__missing_modules = [] self.__missing_namespace = None self.__missing_revision = None self.is_yang_lib = is_yang_lib self.html_result_dir = html_result_dir self.jsons = jsons self.__is_vendor = is_vendor self.revision = '*' self.__path = path self.features = [] self.deviations = [] self.yang_models = yang_models_dir if is_vendor: if is_yang_lib: self.deviations = data['deviations'] self.features = data['features'] self.revision = data['revision'] if self.revision is None: self.revision = '*' self.__path = self.__find_file(data['name'], self.revision) else: self.features = self.\ __resolve_deviations_and_features('features=', data) self.deviations = \ self.__resolve_deviations_and_features('deviations=', data) if 'revision' in data: revision_and_more = data.split('revision=')[1] revision = revision_and_more.split('&')[0] self.revision = revision self.__path = self.__find_file(data.split('&')[0], self.revision) else: self.__path = path if is_vendor_imp_inc: self.__is_vendor = True if self.__path: self.name = None self.organization = None self.ietf_wg = None self.namespace = None self.schema = None self.generated_from = None self.maturity_level = None self.document_name = None self.author_email = None self.reference = None self.tree = None self.expired = None self.expiration_date = None self.module_classification = None self.compilation_status = None self.compilation_result = {} self.prefix = None self.yang_version = None self.description = None self.contact = None self.belongs_to = None self.submodule = [] self.dependencies = [] self.module_type = None self.tree_type = None self.semver = None self.derived_semver = None self.implementation = [] self.imports = [] self.json_submodules = json.dumps([]) self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path)) if self.__parsed_yang is None: raise ParseException(path) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path.split('&')[0])
def __init__(self, yang_models_dir, log_directory, path, html_result_dir, jsons, temp_dir, is_vendor=False, is_yang_lib=False, data=None, is_vendor_imp_inc=False, run_integrity=False): global LOGGER LOGGER = log.get_logger('modules', log_directory + '/parseAndPopulate.log') self.run_integrity = run_integrity self.__temp_dir = temp_dir self.__missing_submodules = [] self.__missing_modules = [] self.__missing_namespace = None self.__missing_revision = None self.is_yang_lib = is_yang_lib self.html_result_dir = html_result_dir self.jsons = jsons self.__is_vendor = is_vendor self.revision = '*' self.__path = path self.features = [] self.deviations = [] self.yang_models = yang_models_dir if is_vendor: if is_yang_lib: self.deviations = data['deviations'] self.features = data['features'] self.revision = data['revision'] if self.revision is None: self.revision = '*' self.__path = self.__find_file(data['name'], self.revision) else: self.features = self.\ __resolve_deviations_and_features('features=', data) self.deviations = \ self.__resolve_deviations_and_features('deviations=', data) if 'revision' in data: revision_and_more = data.split('revision=')[1] revision = revision_and_more.split('&')[0] self.revision = revision self.__path = self.__find_file( data.split('&')[0], self.revision) else: self.__path = path if is_vendor_imp_inc: self.__is_vendor = True if self.__path: self.name = None self.organization = None self.ietf_wg = None self.namespace = None self.schema = None self.generated_from = None self.maturity_level = None self.document_name = None self.author_email = None self.reference = None self.tree = None self.expired = None self.expiration_date = None self.module_classification = None self.compilation_status = None self.compilation_result = {} self.prefix = None self.yang_version = None self.description = None self.contact = None self.belongs_to = None self.submodule = [] self.dependencies = [] self.module_type = None self.tree_type = None self.semver = None self.derived_semver = None self.implementation = [] self.imports = [] self.json_submodules = json.dumps([]) self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path)) if self.__parsed_yang is None: raise ParseException(path) else: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path.split('&')[0])
def add_modules(): """Endpoint is used to add new modules using the API. PUT request is used for updating each module in request body. POST request is used for creating new modules that are not in ConfD/Redis yet. First it checks if the sent request is ok and if so, it will send a another request to the receiver which will work on adding/updating modules while this request will send a "job_id" in the response back to the user. User is able to check success of the job using this "job_id" the job process. :return response with "job_id" that user can use to check whether the job is still running or Failed or Finished successfully. """ if not request.json: abort(400, description= 'bad request - you need to input json body that conforms with' ' module-metadata.yang module. Received no json') body = request.json modules_cont = body.get('modules') if modules_cont is None: abort( 400, description= 'bad request - "modules" json object is missing and is mandatory') module_list = modules_cont.get('module') if module_list is None: abort(400, description= 'bad request - "module" json list is missing and is mandatory') app.logger.info('Adding modules with body\n{}'.format( json.dumps(body, indent=2))) dst_path = os.path.join( ac.d_save_requests, 'sdo-{}.json'.format(datetime.utcnow().strftime(backup_date_format))) if not os.path.exists(ac.d_save_requests): os.mkdir(ac.d_save_requests) with open(dst_path, 'w') as f: json.dump(body, f) response = app.confdService.put_module_metadata(json.dumps(body)) if response.status_code != 200 and response.status_code != 201 and response.status_code != 204: abort( 400, description= 'The body you have provided could not be parsed. ConfD error text:\n{}\n' 'Error code: {}'.format(response.text, response.status_code)) direc_num = 0 while os.path.isdir(os.path.join(ac.d_temp, str(direc_num))): direc_num += 1 direc = os.path.join(ac.d_temp, str(direc_num)) try: os.makedirs(direc) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise repos: t.Dict[str, repoutil.RepoUtil] = {} warning = [] missing_msg = 'bad request - at least one module object is missing mandatory field {}' for module in module_list: app.logger.debug(module) source_file: dict = module.get('source-file') if source_file is None: abort(400, description=missing_msg.format('source-file')) organization_sent = module.get('organization') if organization_sent is None: abort(400, description=missing_msg.format('organization')) name = module.get('name') if name is None: abort(400, description=missing_msg.format('name')) revision = module.get('revision') if revision is None: abort(400, description=missing_msg.format('revision')) if request.method == 'POST': # Check if the module is already in Redis redis_module = get_mod_redis(module) if redis_module != {}: continue module_path = source_file.get('path') if module_path is None: abort(400, description=missing_msg.format('source-file["path"]')) repo_name = source_file.get('repository') if repo_name is None: abort(400, description=missing_msg.format('source-file["repository"]')) owner = source_file.get('owner') if owner is None: abort(400, description=missing_msg.format('source-file["owner"]')) dir_in_repo = os.path.dirname(module_path) repo_url = os.path.join(github_url, owner, repo_name) if repo_url not in repos: repos[repo_url] = get_repo(repo_url, owner, repo_name) # needed to later construct the schema source_file['commit-hash'] = repos[repo_url].get_commit_hash( branch=source_file.get('branch', 'HEAD')) save_to = os.path.join(direc, owner, repo_name.split('.')[0], dir_in_repo) try: os.makedirs(save_to) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise try: shutil.copy(os.path.join(repos[repo_url].local_dir, module_path), save_to) except FileNotFoundError: app.logger.exception('Problem with file {}'.format(module_path)) warning.append('{} does not exist'.format( os.path.join(repo_url, 'blob', source_file['commit-hash'], module_path))) continue organization_parsed = '' try: namespace = yangParser.parse(os.path.abspath('{}/{}'.format(save_to, os.path.basename(module_path)))) \ .search('namespace')[0].arg organization_parsed = organization_by_namespace(namespace) except: while True: try: belongs_to = yangParser.parse(os.path.abspath(os.path.join(repos[repo_url].local_dir, module_path))) \ .search('belongs-to')[0].arg except: break namespace = yangParser.parse( os.path.abspath('{}/{}/{}.yang'.format( repos[repo_url].local_dir, os.path.dirname(module_path), belongs_to))).search('namespace')[0].arg organization_parsed = organization_by_namespace(namespace) break resolved_authorization = authorize_for_sdos(request, organization_sent, organization_parsed) if not resolved_authorization: shutil.rmtree(direc) abort(401, description='Unauthorized for server unknown reason') if 'organization' in repr(resolved_authorization): warning.append('{} {}'.format(os.path.basename(module_path), resolved_authorization)) with open(os.path.join(direc, 'request-data.json'), 'w') as f: json.dump(body, f) arguments = [ 'POPULATE-MODULES', '--sdo', '--dir', direc, '--api', '--credentials', ac.s_confd_credentials[0], ac.s_confd_credentials[1] ] job_id = ac.sender.send('#'.join(arguments)) app.logger.info('Running populate.py with job_id {}'.format(job_id)) if len(warning) > 0: return { 'info': 'Verification successful', 'job-id': job_id, 'warnings': [{ 'warning': val } for val in warning] } else: return ({'info': 'Verification successful', 'job-id': job_id}, 202)