def resolve_organization(path): """Parse yang file and resolve organization out of the module. If the module is a submodule find it's parent and resolve its organization Arguments: :param path: (str) path to a file to parse and resolve a organization :return: list containing amount of yang files and amount that pass compilation respectively """ organization = '' try: namespace = yangParser.parse(os.path.abspath(path)) \ .search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT except: while True: try: belongs_to = yangParser.parse(os.path.abspath(path)) \ .search('belongs-to')[0].arg except: break try: yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang' , belongs_to + '@*.yang') namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT break except: try: yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang' , belongs_to + '@*.yang') namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = MISSING_ELEMENT break except: organization = MISSING_ELEMENT return organization
def check_early_revisions(directory): for f in os.listdir(directory): fname = f.split('.yang')[0].split('@')[0] files_to_delete = [] revisions = [] for f2 in os.listdir(directory): if f2.split('.yang')[0].split('@')[0] == fname: if f2.split(fname)[1].startswith('.') or f2.split( fname)[1].startswith('@'): files_to_delete.append(f2) revision = f2.split(fname)[1].split('.')[0].replace( '@', '') if revision == '': revision = (yangParser.parse( os.path.abspath(directory + f2)).search('revision')[0].arg) year = int(revision.split('-')[0]) month = int(revision.split('-')[1]) day = int(revision.split('-')[2]) revisions.append(datetime(year, month, day)) latest = revisions.index(max(revisions)) files_to_delete.remove(files_to_delete[latest]) for fi in files_to_delete: if 'iana-if-type' in fi: break os.remove(directory + fi)
def check(self, module_name, path, xml_path): module_path = Integrity.find_first_file(path, module_name + '.yang', module_name + '@*.yang') if module_path is None: self.missing_modules[xml_path].add(module_name) return if module_path in self.useless_modules: self.useless_modules.remove(module_path) if module_name not in self.parsed_yang: self.parsed_yang[module_name] = yangParser.parse(os.path.abspath(module_path), self.ctx) namespace = None try: namespace = self.deserialize(self.parsed_yang[module_name].search('namespace')[0].arg) except IndexError: self.missing_wrong_namespaces[xml_path].add(module_name + ': missing_data') try: self.deserialize(self.parsed_yang[module_name].search('revision')[0].arg) except IndexError: self.missing_revision[xml_path].add(module_name) try: imports = self.parsed_yang[module_name].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path, xml_path) except IndexError: pass try: includes = self.parsed_yang[module_name].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path, xml_path) except IndexError: pass if namespace is not None: self.check_namespace(path, namespace, module_name, xml_path)
def check_include(self, include, path, xml_path): module_path = Integrity.find_first_file(path, include + '.yang', include + '@*.yang') if module_path is None: self.missing_submodules[xml_path].add(include) if module_path in self.useless_modules: self.useless_modules.remove(module_path) if include not in self.parsed_yang: self.parsed_yang[include] = yangParser.parse( os.path.abspath(module_path), self.ctx) try: self.deserialize( self.parsed_yang[include].search('revision')[0].arg) except IndexError: self.missing_revision[xml_path].add(include) try: imports = self.parsed_yang[include].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path, xml_path) except IndexError: pass try: includes = self.parsed_yang[include].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path, xml_path) except IndexError: pass
def resolve_revision(yang_file): try: parsed_yang = yangParser.parse(os.path.abspath(yang_file)) revision = parsed_yang.search('revision')[0].arg except: revision = '1970-01-01' return revision
def get_latest_revision(f): stmt = yangParser.parse(f) rev = stmt.search_one('revision') if rev is None: return None return rev.arg
def get_specifics(path_dir): """Get amount of yang files in specified directory and the amount that passed compilation Arguments: :param path_dir: (str) path to directory where we are searching for yang files :return: list containing amount of yang files and amount that pass compilation respectively """ passed = 0 num_in_catalog = 0 for mod_git in list_of_yang_modules_in_subdir(path_dir): try: revision = yangParser.parse( os.path.abspath(mod_git)).search('revision')[0].arg except: continue organization = resolve_organization(mod_git) name = mod_git.split('/')[-1].split('.')[0].split('@')[0] if revision is None: revision = '1500-01-01' if name is None or organization is None: continue if ',' in organization: organization = organization.replace(' ', '%20') path = yangcatalog_api_prefix + 'search/name/' + name module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() org = json.loads( data)['yang-catalog:modules']['module'][0]['organization'] rev = json.loads( data)['yang-catalog:modules']['module'][0]['revision'] status = json.loads(data)['yang-catalog:modules']['module'][0][ 'compilation-status'] if org == organization and rev == revision: if 'passed' == status: passed += 1 num_in_catalog += 1 else: organization = organization.replace(' ', '%20') path = yangcatalog_api_prefix + 'search/modules/' + name + ','\ + revision + ',' + organization module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() if 'passed' == json.loads( data)['module'][0]['compilation-status']: passed += 1 num_in_catalog += 1 return [num_in_catalog, passed]
def search_dependencies(base): for root, dirs, files in os.walk(base): for basename in files: if fnmatch.fnmatch(basename, '*.yang'): filename = os.path.join(root, basename) LOGGER.info('Parsing file {}'.format(filename)) try: yangFile = yangParser.parse(os.path.abspath(filename)) try: revision = yangFile.search('revision')[0].arg except: revision = '1970-01-01' name = filename.split('/')[-1].split('.')[0].split('@')[0] key = '{}@{}.yang'.format(name, revision) if key not in dependencies: dependencies[key] = set() yangImport = yangFile.search('import') yangInclude = yangFile.search('include') for imp in yangImport: impName = imp.arg impRev = None for sub in imp.substmts: if sub.keyword == 'revision-date': impRev = sub.arg if impRev: name_rev = '{}@{}'.format(impName, impRev) dependencies[key].add(name_rev) if name_rev not in dependants: dependants[name_rev] = set() dependants[name_rev].add(key) else: dependencies[key].add(impName) if impName not in dependants: dependants[impName] = set() dependants[impName].add(key) for inc in yangInclude: incName = inc.arg incRev = None for sub in inc.substmts: if sub.keyword == 'revision-date': incRev = sub.arg if incRev: name_rev = '{}@{}'.format(incName, incRev) dependencies[key].add(name_rev) if name_rev not in dependants: dependants[name_rev] = set() dependants[name_rev].add(key) else: dependencies[key].add(incName) if incName not in dependants: dependants[incName] = set() dependants[incName].add(key) except: pass
def resolve_organization(path): organization = '' try: namespace = yangParser.parse(os.path.abspath(path)) \ .search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = yangParser.parse( os.path.abspath(path)).search('organization')[0].arg except: while True: try: belongs_to = yangParser.parse(os.path.abspath(path)) \ .search('belongs-to')[0].arg except: break try: yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang', belongs_to + '@*.yang') namespace = yangParser.parse( os.path.abspath(yang_file)).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] if organization == '': organization = yangParser.parse( os.path.abspath(path)).search('organization')[0].arg break except: pass return organization
def check_name_no_revision_exist(directory): LOGGER.info('Checking revision for directory: {}'.format(directory)) for root, dirs, files in os.walk(directory): for basename in files: if '@' in basename: yang_file_name = basename.split('@')[0] + '.yang' revision = basename.split('@')[1].split('.')[0] exists = os.path.exists(directory + yang_file_name) if exists: parsed_yang = yangParser.parse(os.path.abspath(directory + yang_file_name)) comapred_revision = parsed_yang.search('revision')[0].arg if revision == comapred_revision: os.remove(directory + yang_file_name)
def __resolve_submodule(self): try: submodules = self.__parsed_yang.search('include') except: return if len(submodules) == 0: return for chunk in submodules: dep = self.Dependencies() sub = self.Submodules() sub.name = chunk.arg if len(chunk.search('revision-date')) > 0: sub.revision = chunk.search('revision-date')[0].arg if sub.revision: yang_file = self.__find_file(sub.name, sub.revision, True) dep.revision = sub.revision else: yang_file = self.__find_file(sub.name, submodule=True) try: sub.revision = \ yangParser.parse(os.path.abspath(yang_file)).search( 'revision')[0].arg except: sub.revision = '1970-01-01' if yang_file is None: LOGGER.error('Module can not be found') continue path = '/'.join(self.schema.split('/')[0:-1]) path += '/{}'.format(yang_file.split('/')[-1]) if yang_file: sub.schema = path dep.name = sub.name dep.schema = sub.schema self.dependencies.append(dep) self.submodule.append(sub) self.json_submodules = json.dumps([{ 'name': self.submodule[x].name, 'schema': self.submodule[x].schema, 'revision': self.submodule[x].revision } for x in range(0, len(self.submodule))])
def get_specifics(path_dir): """Get amount of yang files in specified directory and the amount that passed compilation Arguments: :param path_dir: (str) path to directory where we are searching for yang files :return: list containing amount of yang files and amount that pass compilation respectively """ passed = 0 num_in_catalog = 0 for mod_git in list_of_yang_modules_in_subdir(path_dir): try: revision = yangParser.parse(os.path.abspath(mod_git)).search('revision')[0].arg except: continue organization = resolve_organization(mod_git) name = mod_git.split('/')[-1].split('.')[0].split('@')[0] if revision is None: revision = '1500-01-01' if name is None or organization is None: continue if ',' in organization: organization = organization.replace(' ', '%20') path = yangcatalog_api_prefix + 'search/name/' + name module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() org = json.loads(data)['yang-catalog:modules']['module'][0]['organization'] rev = json.loads(data)['yang-catalog:modules']['module'][0]['revision'] status = json.loads(data)['yang-catalog:modules']['module'][0]['compilation-status'] if org == organization and rev == revision: if 'passed' == status: passed += 1 num_in_catalog += 1 else: organization = organization.replace(' ', '%20') path = yangcatalog_api_prefix + 'search/modules/' + name + ','\ + revision + ',' + organization module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() if 'passed' == json.loads(data)['module'][0]['compilation-status']: passed += 1 num_in_catalog += 1 return [num_in_catalog, passed]
def get_specifics(path_dir): passed = 0 num_in_catalog = 0 for mod_git in list_of_yang_modules_in_subdir(path_dir): try: revision = yangParser.parse( os.path.abspath(mod_git)).search('revision')[0].arg except: continue organization = resolve_organization(mod_git) name = mod_git.split('/')[-1].split('.')[0].split('@')[0] if revision is None: revision = '1500-01-01' if name is None or organization is None: continue if ',' in organization: path = protocol + '://' + api_ip + ':' + api_port + '/search/name/' + name module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() org = json.loads( data)['yang-catalog:modules']['module'][0]['organization'] rev = json.loads( data)['yang-catalog:modules']['module'][0]['revision'] status = json.loads(data)['yang-catalog:modules']['module'][0][ 'compilation-status'] if org == organization and rev == revision: if 'passed' == status: passed += 1 num_in_catalog += 1 else: path = protocol + '://' + api_ip + ':' + api_port + '/search/modules/' + name + ',' + revision + ',' + \ organization module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json') if module_exist: data = module_exist.read() if 'passed' == json.loads( data)['module'][0]['compilation-status']: passed += 1 num_in_catalog += 1 return [num_in_catalog, passed]
def add_vendor_information(self, vendor, platform_data, software_version, os_version, feature_set, os_type, confarmance_type, capability, netconf_version, integrity_checker, split): for data in platform_data: implementation = self.Implementations() implementation.vendor = vendor implementation.platform = data['platform'] implementation.software_version = software_version implementation.software_flavor = data['software-flavor'] implementation.os_version = os_version implementation.feature_set = feature_set implementation.os_type = os_type implementation.feature = self.features implementation.capability = capability implementation.netconf_version = netconf_version if self.is_yang_lib: for x in range(0, len(self.deviations['name'])): devs = implementation.Deviations() devs.name = self.deviations['name'][x] devs.revision = self.deviations['revision'][x] implementation.deviations.append(devs) else: for name in self.deviations: devs = implementation.Deviations() devs.name = name yang_file = self.__find_file(name) if yang_file is None: devs.revision = '1970-01-01' else: try: s = yang_file.split('/') key = '/'.join(split[0:-1]) integrity_checker.remove_one(key, s[-1]) devs.revision = yangParser.parse(os.path.abspath(yang_file)) \ .search('revision')[0].arg except: devs.revision = '1970-01-01' implementation.deviations.append(devs) implementation.conformance_type = confarmance_type self.implementation.append(implementation)
def find_first_file(directory, pattern, pattern_with_revision): for root, dirs, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern_with_revision): filename = os.path.join(root, basename) return filename for root, dirs, files in os.walk(directory): for basename in files: if fnmatch.fnmatch(basename, pattern): filename = os.path.join(root, basename) try: revision = yangParser.parse(filename).search('revision')[0]\ .arg except: revision = '1970-01-01' if '*' not in pattern_with_revision: if revision in pattern_with_revision: return filename else: return filename
def check(self, module_name, path, xml_path): module_path = Integrity.find_first_file(path, module_name + '.yang', module_name + '@*.yang') if module_path is None: self.missing_modules[xml_path].add(module_name) return if module_path in self.useless_modules: self.useless_modules.remove(module_path) if module_name not in self.parsed_yang: self.parsed_yang[module_name] = yangParser.parse( os.path.abspath(module_path), self.ctx) namespace = None try: namespace = self.deserialize( self.parsed_yang[module_name].search('namespace')[0].arg) except IndexError: self.missing_wrong_namespaces[xml_path].add(module_name + ': missing_data') try: self.deserialize( self.parsed_yang[module_name].search('revision')[0].arg) except IndexError: self.missing_revision[xml_path].add(module_name) try: imports = self.parsed_yang[module_name].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path, xml_path) except IndexError: pass try: includes = self.parsed_yang[module_name].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path, xml_path) except IndexError: pass if namespace is not None: self.check_namespace(path, namespace, module_name, xml_path)
def __resolve_submodule_case(self, field): if self.module_type == 'submodule': LOGGER.debug( 'Getting parent information because file {} is a submodule'. format(self.__path)) yang_file = self.__find_file(self.belongs_to) if yang_file is None: return None parsed_parent_yang = yangParser.parse(os.path.abspath(yang_file)) try: return parsed_parent_yang.search(field)[0].arg except: if field == 'prefix': return None else: return MISSING_ELEMENT else: try: return self.__parsed_yang.search(field)[0].arg except: if field == 'prefix': return None else: return MISSING_ELEMENT
def check_early_revisions(directory): for f in os.listdir(directory): fname = f.split('.yang')[0].split('@')[0] files_to_delete = [] revisions = [] for f2 in os.listdir(directory): if f2.split('.yang')[0].split('@')[0] == fname: if f2.split(fname)[1].startswith('.') or f2.split(fname)[1].startswith('@'): files_to_delete.append(f2) revision = f2.split(fname)[1].split('.')[0].replace('@', '') if revision == '': revision = (yangParser.parse(os.path.abspath(directory + f2)) .search('revision')[0].arg) year = int(revision.split('-')[0]) month = int(revision.split('-')[1]) day = int(revision.split('-')[2]) revisions.append(datetime(year, month, day)) latest = revisions.index(max(revisions)) files_to_delete.remove(files_to_delete[latest]) for fi in files_to_delete: if 'iana-if-type' in fi: break os.remove(directory + fi)
def check_include(self, include, path, xml_path): module_path = Integrity.find_first_file(path, include + '.yang', include + '@*.yang') if module_path is None: self.missing_submodules[xml_path].add(include) if module_path in self.useless_modules: self.useless_modules.remove(module_path) if include not in self.parsed_yang: self.parsed_yang[include] = yangParser.parse(os.path.abspath(module_path), self.ctx) try: self.deserialize(self.parsed_yang[include].search('revision')[0].arg) except IndexError: self.missing_revision[xml_path].add(include) try: imports = self.parsed_yang[include].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path, xml_path) except IndexError: pass try: includes = self.parsed_yang[include].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path, xml_path) except IndexError: pass
def add_modules(): if not request.json: abort(400) body = request.json LOGGER.info('Adding modules with body {}'.format(body)) tree_created = False with open('./prepare-sdo.json', "w") as plat: json.dump(body, plat) try: http_request( protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/modules/', 'DELETE', None, credentials, 'application/vnd.yang.collection+json') except: pass path = protocol + '://' + confd_ip + ':' + repr( confdPort) + '/api/config/modules' try: http_request(path, 'PUT', json.dumps(body), credentials, 'application/vnd.yang.data+json') except urllib2.HTTPError as e: abort(e.code) repo = {} warning = [] direc = 0 while True: if os.path.isdir('../parseAndPopulate/' + repr(direc)): direc += 1 else: break direc = '../parseAndPopulate/' + repr(direc) for mod in body['modules']['module']: sdo = mod['source-file'] orgz = mod['organization'] if request.method == 'POST': try: path = protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/catalog/modules/module/' + \ mod['name'] + ',' + mod['revision'] + ',' + mod['organization'] http_request(path, 'GET', None, credentials, 'application/vnd.yang.data+json') continue except urllib2.HTTPError as e: pass directory = '/'.join(sdo['path'].split('/')[:-1]) repo_url = url + sdo['owner'] + '/' + sdo['repository'] LOGGER.debug('Cloning repository') if repo_url not in repo: repo[repo_url] = repoutil.RepoUtil(repo_url) repo[repo_url].clone() for submodule in repo[repo_url].repo.submodules: submodule.update(init=True) if sdo.get('branch'): branch = sdo.get('branch') else: branch = 'master' save_to = direc + '/temp/' + sdo['owner'] + '/' + sdo['repository'].split('.')[0] \ + '/' + branch + '/' + directory try: os.makedirs(save_to) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise shutil.copy(repo[repo_url].localdir + '/' + sdo['path'], save_to) if os.path.isfile('./prepare-sdo.json'): shutil.move('./prepare-sdo.json', direc) tree_created = True organization = '' try: namespace = yangParser.parse(os.path.abspath(save_to + '/' + sdo['path'].split('/')[-1])) \ .search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split(':')[0] except: while True: try: belongs_to = yangParser.parse(os.path.abspath(repo[repo_url].localdir + '/' + sdo['path'])) \ .search('belongs-to')[0].arg except: break try: namespace = yangParser.parse( os.path.abspath(repo[repo_url].localdir + '/' + '/'.join(sdo['path'].split('/')[:-1]) + '/' + belongs_to + '.yang')).search('namespace')[0].arg for ns, org in NS_MAP.items(): if ns in namespace: organization = org if organization == '': if 'urn:' in namespace: organization = namespace.split('urn:')[1].split( ':')[0] break except: pass resolved_authorization = authorize_for_sdos(request, orgz, organization) if not resolved_authorization: shutil.rmtree(direc) for key in repo: repo[key].remove() return unauthorized() if 'organization' in repr(resolved_authorization): warning.append(sdo['path'].split('/')[-1] + ' ' + resolved_authorization) for key in repo: repo[key].remove() api_protocol = 'http' api_port = 5000 if ssl_key != '': api_protocol = 'https' api_port = 8443 LOGGER.debug('Sending a new job') arguments = [ "python", "../parseAndPopulate/populate.py", "--sdo", "--port", repr(confdPort), "--dir", direc + "/temp", "--api", "--ip", confd_ip, "--credentials", credentials[0], credentials[1], repr(tree_created), protocol, api_protocol, repr(api_port) ] job_id = sender.send('#'.join(arguments)) LOGGER.info('job_id {}'.format(job_id)) if len(warning) > 0: return jsonify({ 'info': 'Verification successful', 'job-id': job_id, 'warnings': [{ 'warning': val } for val in warning] }) else: return jsonify({'info': 'Verification successful', 'job-id': job_id})
def __init__(self, path, html_result_dir, jsons, temp_dir, is_vendor=False, is_yang_lib=False, data=None, is_vendor_imp_inc=False, run_integrity=False): self.run_integrity = run_integrity self.__temp_dir = temp_dir self.__missing_submodules = [] self.__missing_modules = [] self.__missing_namespace = None self.__missing_revision = None self.is_yang_lib = is_yang_lib self.html_result_dir = html_result_dir self.jsons = jsons self.__is_vendor = is_vendor self.revision = '*' self.__path = path self.features = [] self.deviations = [] if is_vendor: if is_yang_lib: self.deviations = data['deviations'] self.features = data['features'] self.revision = data['revision'] if self.revision is None: self.revision = '*' self.__path = self.__find_file(data['name'], self.revision) else: self.features = self.\ __resolve_deviations_and_features('features=', data) self.deviations = \ self.__resolve_deviations_and_features('deviations=', data) if 'revision' in data: revision_and_more = data.split('revision=')[1] revision = revision_and_more.split('&')[0] self.revision = revision self.__path = self.__find_file( data.split('&')[0], self.revision) else: self.__path = path if is_vendor_imp_inc: self.__is_vendor = True if self.__path: self.name = None self.organization = None self.ietf_wg = None self.namespace = None self.schema = None self.generated_from = None self.maturity_level = None self.document_name = None self.author_email = None self.reference = None self.tree = None self.expired = None self.expiration_date = None self.module_classification = None self.compilation_status = None self.compilation_result = {} self.prefix = None self.yang_version = None self.description = None self.contact = None self.belongs_to = None self.submodule = [] self.dependencies = [] self.module_type = None self.tree_type = None self.semver = None self.derived_semver = None self.implementation = [] self.imports = [] self.json_submodules = json.dumps([]) self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path)) if self.__parsed_yang is None: pass # TODO file has wrong format. probably end with CODE END else: raise FileError(path.split('&')[0], 'File not found')