def find_yang_var(self, object, field, module_name, yang_file): try: # In case it is import, include or feature there might be more than one object if 'import' in field or 'include' in field or 'feature' in field: # If this yang is not yet found and parsed if self.parsed_yang is None: # Find and Parse yang file self.parsed_yang = yangParser.parse(os.path.abspath(yang_file)) object[module_name] = [] names = [] revs = [] incl = {} # Search for the field in yang file for chunk in self.parsed_yang.search(field): # If it is included parse it to name and revision if 'include' in field: names.append(chunk.arg) if len(chunk.search('revision-date')) > 0: revs.append(chunk.search('revision-date')[0].arg) # Otherwise add object else: object[module_name].append(chunk.arg) if 'include' in field: incl['name'] = names incl['revision'] = revs object[module_name] = incl else: # If this yang is not yet found and parsed if self.parsed_yang is None: # Find and Parse yang file self.parsed_yang = yangParser.parse(os.path.abspath(yang_file)) # Search for the field and parse first one you find (in case of revision # we want just first one which is the newest one) yang_variable = self.parsed_yang.search(field)[0].arg if isinstance(yang_variable, unicode): object[module_name] = unicodedata.normalize('NFKD', yang_variable).encode('ascii', 'ignore') else: object[module_name] = yang_variable # In case of exception hande them except AttributeError: self.handle_exception(field, object, module_name) except IndexError: self.handle_exception(field, object, module_name) except UnicodeDecodeError: self.handle_exception(field, object, module_name)
def check_include(self, include, path, xml_path): module_path = Integrity.find_first_file(path, include + '.yang', include + '@*.yang') if module_path is None: self.missing_submodules[xml_path].add(include) if module_path in self.useless_modules: self.useless_modules.remove(module_path) if include not in self.parsed_yang: self.parsed_yang[include] = yangParser.parse( os.path.abspath(module_path), self.ctx) try: self.deserialize( self.parsed_yang[include].search('revision')[0].arg) except IndexError: self.missing_revision[xml_path].add(include) try: imports = self.parsed_yang[include].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path, xml_path) except IndexError: pass try: includes = self.parsed_yang[include].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path, xml_path) except IndexError: pass
def check(self, module_name, path): module_path = Integrity.find_first_file(path, module_name + '.yang', module_name + '@*.yang') if module_path is None: self.missing_modules.add(module_name) return if module_path in self.useless_modules: self.useless_modules.remove(module_path) if module_name not in self.parsed_yang: self.parsed_yang[module_name] = yangParser.parse(os.path.abspath(module_path), self.ctx) namespace = None try: namespace = self.deserialize(self.parsed_yang[module_name].search('namespace')[0].arg) except IndexError: self.missing_wrong_namespaces.add(module_name + ': missing_data') try: self.deserialize(self.parsed_yang[module_name].search('revision')[0].arg) except IndexError: self.missing_revision.add(module_name) try: imports = self.parsed_yang[module_name].search('import') for imp in imports: self.check(self.deserialize(imp.arg), path) except IndexError: pass try: includes = self.parsed_yang[module_name].search('include') for include in includes: self.check_include(self.deserialize(include.arg), path) except IndexError: pass if namespace is not None: self.check_namespace(path, namespace, module_name)
def add_modules(): if not request.json: abort(400) body = request.json with open('./prepare-sdo.json', "w") as plat: json.dump(body, plat) repo = {} warning = [] for mod in body['modules']['module']: sdo = mod['source-file'] org = mod['organization'] directory = '/'.join(sdo['path'].split('/')[:-1]) repo_url = url + sdo['owner'] + '/' + sdo['repository'] if repo_url not in repo: repo[repo_url] = repoutil.RepoUtil(repo_url) repo[repo_url].clone() for submodule in repo[repo_url].repo.submodules: submodule.update(init=True) save_to = 'temp/' + sdo['owner'] + '/' + sdo['repository'].split('.')[0] + '/' + directory try: os.makedirs(save_to) except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise shutil.copy(repo[repo_url].localdir + '/' + sdo['path'], save_to) organization = yangParser.parse(os.path.abspath(save_to + '/' + sdo['path'].split('/')[-1])) \ .search('organization')[0].arg resolved_authorization = authorize_for_sdos(request, org, organization) if not resolved_authorization: shutil.rmtree('temp/') for key in repo: repo[key].remove() return unauthorized() if 'organization' in repr(resolved_authorization): warning.append(sdo['path'].split('/')[-1] + ' ' + resolved_authorization) with open("log.txt", "wr") as f: try: arguments = ["python", "../parseAndPopulate/populate.py", "--sdo", "--port", repr(confdPort), "--dir", "temp", "--api", "--ip", confd_ip, "--credentials", credentials[0], credentials[1]] subprocess.check_call(arguments, stderr=f) except subprocess.CalledProcessError as e: shutil.rmtree('temp/') for key in repo: repo[key].remove() return jsonify({'error': 'Was not able to write all or partial yang files'}) try: os.makedirs('../../api/sdo/') except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise subprocess.call(["cp", "-r", "temp/.", "../../api/sdo/"]) shutil.rmtree('temp') for item in os.listdir('./'): if 'log' in item and '.txt' in item: os.remove(item) for key in repo: repo[key].remove() if len(warning) > 0: return jsonify({'info': 'success', 'warnings': [{'warning': val}for val in warning]}) else: return jsonify({'info': 'success'})
def parse_and_dump(self): capability = [] tag = self.root.tag module_names = [] name_revision = [] deviations = {} features = {} revision = {} yang_version = {} namespace = {} prefix = {} organization = {} contact = {} description = {} includes = {} schema = {} imports = {} reference = {} conformance_type = {} compilations_status = {} working_group = {} author_email = {} netconf_version = '' compilations_result = {} organization_module = {} # Parse deviations and features from each module from netconf hello message def deviations_and_features(search_for): my_list = [] if search_for in module_and_more: devs_or_features = module_and_more.split(search_for)[1] devs_or_features = devs_or_features.split('&')[0] my_list = devs_or_features.split(',') return my_list # netconf capability parsing for cap in self.root.iter(tag.split('hello')[0] + 'capability'): # Parse netconf version if ':netconf:base:' in cap.text: netconf_version = cap.text # Parse capability together with version if ':capability:' in cap.text: cap_with_version = cap.text.split(':capability:')[1] capability.append(cap_with_version.split('?')[0]) # Parse modules if 'module=' in cap.text: # Parse name of the module module_and_more = cap.text.split('module=')[1] module_name = module_and_more.split('&')[0] module_names.append(module_name) devs = {} revs = [] # Parse deviations of the module names = deviations_and_features('deviations=') # Find and parse deviated yang file so we can get a revision out of it for i in names: yang_file = find_first_file('/'.join(self.split[0:-1]), i + '.yang', i + '.yang') if yang_file is None: self.integrity_checker.add_module(self.split, i) revs.append('1500-01-01') else: self.parsed_yang = yangParser.parse(os.path.abspath(yang_file)) yang_variable = self.parsed_yang.search('revision')[0].arg if isinstance(yang_variable, unicode): revs.append(unicodedata.normalize('NFKD', yang_variable).encode('ascii', 'ignore')) else: revs.append(yang_variable) if yang_file is not None: self.integrity_checker.remove_one(self.split, yang_file) devs['name'] = names devs['revision'] = revs deviations[module_name] = devs # Parse features of the module features[module_name] = deviations_and_features('features=') # Parse conformance type of the module conformance_type[module_name] = 'implement' # Parse revision of the module from capability.xml file my_var = '' if 'revision' in cap.text: revision_and_more = cap.text.split('revision=')[1] my_var = revision_and_more.split('&')[0] revision[module_name] = my_var else: yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang', module_name + '.yang') self.find_yang_var(revision, 'revision', module_name, yang_file) name_revision.append(module_name + '@' + revision[module_name]) # Find yang file in the same directory as capability.xml file is # so we can parse all needed fields out of it yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang', module_name + '@' + my_var + '.yang') if yang_file is None: # In case we didn`t find the module try to look for it in any other directory of this project self.integrity_checker.add_module(self.split, module_name) yang_file = find_first_file('/'.join(self.split[0:1]), module_name + '.yang', module_name + '@' + my_var + '.yang') if yang_file is not None: self.integrity_checker.remove_one(self.split, yang_file) self.parsed_yang = None namespace_exist = False # Parse rest of the fields out of the yang file self.find_yang_var(namespace, 'namespace', module_name, yang_file) for ns, org in NS_MAP.items(): if self.os_version is '1651': if ns is 'urn:cisco': if ns in namespace[module_name]: organization_module[module_name] = org namespace_exist = True else: if ns in namespace[module_name]: organization_module[module_name] = org namespace_exist = True if not namespace_exist: organization_module[module_name] = 'missing_data' if namespace[module_name] is None: self.integrity_checker.add_namespace(self.split, module_name + ' : missing data') namespace[module_name] = 'missing data' self.integrity_checker.add_namespace(self.split, module_name + ' : ' + namespace[module_name]) self.find_yang_var(prefix, 'prefix', module_name, yang_file) self.find_yang_var(yang_version, 'yang-version', module_name, yang_file) self.find_yang_var(organization, 'organization', module_name, yang_file) self.find_yang_var(contact, 'contact', module_name, yang_file) self.find_yang_var(description, 'description', module_name, yang_file) self.find_yang_var(includes, 'include', module_name, yang_file) self.find_yang_var(imports, 'import', module_name, yang_file) self.find_yang_var(reference, 'reference', module_name, yang_file) self.find_yang_var(schema, 'schema', module_name, yang_file) compilations_status[module_name] = self.parse_status(module_name, revision[module_name]) if compilations_status[module_name] not in 'PASSED': compilations_result[module_name] = self.parse_result(module_name, revision[module_name]) else: compilations_result[module_name] = '' author_email[module_name] = self.parse_email(module_name, revision[module_name]) working_group[module_name] = self.parse_wg(module_name, revision[module_name]) self.prepare.add_key(module_name + '@' + revision[module_name] + '@' + namespace[module_name] + '@' + conformance_type[module_name]) self.parse_imports_includes(includes[module_name]['name'], features, revision, name_revision, yang_version, namespace, prefix, organization, contact, description, includes, imports, reference, conformance_type, deviations, module_names, compilations_status, schema, author_email, working_group, organization_module, True, namespace[module_name], compilations_result) self.parse_imports_includes(imports[module_name], features, revision, name_revision, yang_version, namespace, prefix, organization, contact, description, includes, imports, reference, conformance_type, deviations, module_names, compilations_status, schema, author_email, working_group, organization_module, False, namespace[module_name], compilations_result) # restconf capability parsing for cap in self.root.iter('module'): module_name = cap.find('name').text module_names.append(module_name) namespace[module_name] = cap.find('namespace').text revision[module_name] = cap.find('revision').text schema[module_name] = cap.find('schema').text conformance_type[module_name] = cap.find('conformance-type').text devs = {} names = [] revs = [] for dev in self.root.iter('deviation'): names.append(dev.find('name').text) if dev.find('revision').text is None: revs.append('1500-01-01') else: revs.append(dev.find('revision').text) devs['name'] = names devs['revision'] = revs deviations[module_name] = devs objs = [] for feat in self.root.iter('feature'): objs.append(feat.text) features[module_name] = objs # Find yang file in the same directory as capability.xml file is # so we can parse all needed fields out of it yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang', module_name + '@' + revision[module_name] + '.yang') if yang_file is None: # In case we didn`t find the module try to look for it in any other directory of this project self.integrity_checker.add_module(self.split, module_name) yang_file = find_first_file('/'.join(self.split[0:2]), module_name + '.yang', module_name + '@' + revision[module_name] + '.yang') # Parse rest of the fields out of the yang file self.find_yang_var(prefix, 'prefix', module_name, yang_file) self.find_yang_var(yang_version, 'yang-version', module_name, yang_file) self.find_yang_var(organization, 'organization', module_name, yang_file) self.find_yang_var(contact, 'contact', module_name, yang_file) self.find_yang_var(description, 'description', module_name, yang_file) self.find_yang_var(reference, 'reference', module_name, yang_file) self.find_yang_var(includes, 'include', module_name, yang_file) self.find_yang_var(imports, 'import', module_name, yang_file) self.integrity_checker.add_unique(name_revision) # Write dictionary to file # Create json dictionary out of parsed information with open('normal' + repr(self.index) + '.json', "w") as ietf_mode: json.dump({'vendor': self.vendor, 'os-type': self.os, 'os-version': self.os_version, 'platform': self.platform, 'feature-set': 'ALL', 'protocols': { 'protocol': [{ 'name': 'netconf', 'capabilities': capability, 'protocol-version': netconf_version, }] }, 'modules': { 'module': [ { 'reference': reference.get(module_names[k]), 'prefix': prefix.get(module_names[k]), 'yang-version': yang_version.get(module_names[k]), 'organization': organization_module.get(module_names[k]), 'description': description.get(module_names[k]), 'contact': contact.get(module_names[k]), 'submodule': json.loads( self.get_submodule_info(includes[module_names[k]]['name'])), # 'imports': json.loads( # self.get_submodule_info(imports[module_names[k]]['name'], missing_module, # missing_includes)), 'conformance-type': conformance_type.get(module_names[k]), 'compilation-status': compilations_status.get(module_names[k]), 'author-email': author_email.get(module_names[k]), 'revision': revision.get(module_names[k]), 'namespace': namespace.get(module_names[k]), 'name': module_names[k], 'schema': schema.get(module_names[k]), 'feature': features.get(module_names[k]), 'maturity-level': working_group.get(module_names[k]), 'compilation-result': compilations_result.get(module_names[k]), 'deviation': [ {'name': deviations[module_names[k]]['name'][i], 'revision': deviations[module_names[k]]['revision'][i] } for i, val in enumerate(deviations.get(module_names[k])['name'])], } for k, val in enumerate(module_names)] } }, ietf_mode)