def find_yang_var(self, object, field, module_name, yang_file):
try:
# In case it is import, include or feature there might be more than one object
if 'import' in field or 'include' in field or 'feature' in field:
# If this yang is not yet found and parsed
if self.parsed_yang is None:
# Find and Parse yang file
self.parsed_yang = yangParser.parse(os.path.abspath(yang_file))
object[module_name] = []
names = []
revs = []
incl = {}
# Search for the field in yang file
for chunk in self.parsed_yang.search(field):
# If it is included parse it to name and revision
if 'include' in field:
names.append(chunk.arg)
if len(chunk.search('revision-date')) > 0:
revs.append(chunk.search('revision-date')[0].arg)
# Otherwise add object
else:
object[module_name].append(chunk.arg)
if 'include' in field:
incl['name'] = names
incl['revision'] = revs
object[module_name] = incl
else:
# If this yang is not yet found and parsed
if self.parsed_yang is None:
# Find and Parse yang file
self.parsed_yang = yangParser.parse(os.path.abspath(yang_file))
# Search for the field and parse first one you find (in case of revision
# we want just first one which is the newest one)
yang_variable = self.parsed_yang.search(field)[0].arg
if isinstance(yang_variable, unicode):
object[module_name] = unicodedata.normalize('NFKD', yang_variable).encode('ascii', 'ignore')
else:
object[module_name] = yang_variable
# In case of exception hande them
except AttributeError:
self.handle_exception(field, object, module_name)
except IndexError:
self.handle_exception(field, object, module_name)
except UnicodeDecodeError:
self.handle_exception(field, object, module_name)
def check_include(self, include, path, xml_path):
module_path = Integrity.find_first_file(path, include + '.yang',
include + '@*.yang')
if module_path is None:
self.missing_submodules[xml_path].add(include)
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if include not in self.parsed_yang:
self.parsed_yang[include] = yangParser.parse(
os.path.abspath(module_path), self.ctx)
try:
self.deserialize(
self.parsed_yang[include].search('revision')[0].arg)
except IndexError:
self.missing_revision[xml_path].add(include)
try:
imports = self.parsed_yang[include].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path, xml_path)
except IndexError:
pass
try:
includes = self.parsed_yang[include].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path,
xml_path)
except IndexError:
pass
def check(self, module_name, path):
module_path = Integrity.find_first_file(path, module_name + '.yang', module_name + '@*.yang')
if module_path is None:
self.missing_modules.add(module_name)
return
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if module_name not in self.parsed_yang:
self.parsed_yang[module_name] = yangParser.parse(os.path.abspath(module_path), self.ctx)
namespace = None
try:
namespace = self.deserialize(self.parsed_yang[module_name].search('namespace')[0].arg)
except IndexError:
self.missing_wrong_namespaces.add(module_name + ': missing_data')
try:
self.deserialize(self.parsed_yang[module_name].search('revision')[0].arg)
except IndexError:
self.missing_revision.add(module_name)
try:
imports = self.parsed_yang[module_name].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path)
except IndexError:
pass
try:
includes = self.parsed_yang[module_name].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path)
except IndexError:
pass
if namespace is not None:
self.check_namespace(path, namespace, module_name)
def add_modules():
if not request.json:
abort(400)
body = request.json
with open('./prepare-sdo.json', "w") as plat:
json.dump(body, plat)
repo = {}
warning = []
for mod in body['modules']['module']:
sdo = mod['source-file']
org = mod['organization']
directory = '/'.join(sdo['path'].split('/')[:-1])
repo_url = url + sdo['owner'] + '/' + sdo['repository']
if repo_url not in repo:
repo[repo_url] = repoutil.RepoUtil(repo_url)
repo[repo_url].clone()
for submodule in repo[repo_url].repo.submodules:
submodule.update(init=True)
save_to = 'temp/' + sdo['owner'] + '/' + sdo['repository'].split('.')[0] + '/' + directory
try:
os.makedirs(save_to)
except OSError as e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
shutil.copy(repo[repo_url].localdir + '/' + sdo['path'], save_to)
organization = yangParser.parse(os.path.abspath(save_to + '/' + sdo['path'].split('/')[-1])) \
.search('organization')[0].arg
resolved_authorization = authorize_for_sdos(request, org, organization)
if not resolved_authorization:
shutil.rmtree('temp/')
for key in repo:
repo[key].remove()
return unauthorized()
if 'organization' in repr(resolved_authorization):
warning.append(sdo['path'].split('/')[-1] + ' ' + resolved_authorization)
with open("log.txt", "wr") as f:
try:
arguments = ["python", "../parseAndPopulate/populate.py", "--sdo", "--port", repr(confdPort), "--dir",
"temp", "--api", "--ip", confd_ip, "--credentials", credentials[0], credentials[1]]
subprocess.check_call(arguments, stderr=f)
except subprocess.CalledProcessError as e:
shutil.rmtree('temp/')
for key in repo:
repo[key].remove()
return jsonify({'error': 'Was not able to write all or partial yang files'})
try:
os.makedirs('../../api/sdo/')
except OSError as e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
subprocess.call(["cp", "-r", "temp/.", "../../api/sdo/"])
shutil.rmtree('temp')
for item in os.listdir('./'):
if 'log' in item and '.txt' in item:
os.remove(item)
for key in repo:
repo[key].remove()
if len(warning) > 0:
return jsonify({'info': 'success', 'warnings': [{'warning': val}for val in warning]})
else:
return jsonify({'info': 'success'})
def parse_and_dump(self):
capability = []
tag = self.root.tag
module_names = []
name_revision = []
deviations = {}
features = {}
revision = {}
yang_version = {}
namespace = {}
prefix = {}
organization = {}
contact = {}
description = {}
includes = {}
schema = {}
imports = {}
reference = {}
conformance_type = {}
compilations_status = {}
working_group = {}
author_email = {}
netconf_version = ''
compilations_result = {}
organization_module = {}
# Parse deviations and features from each module from netconf hello message
def deviations_and_features(search_for):
my_list = []
if search_for in module_and_more:
devs_or_features = module_and_more.split(search_for)[1]
devs_or_features = devs_or_features.split('&')[0]
my_list = devs_or_features.split(',')
return my_list
# netconf capability parsing
for cap in self.root.iter(tag.split('hello')[0] + 'capability'):
# Parse netconf version
if ':netconf:base:' in cap.text:
netconf_version = cap.text
# Parse capability together with version
if ':capability:' in cap.text:
cap_with_version = cap.text.split(':capability:')[1]
capability.append(cap_with_version.split('?')[0])
# Parse modules
if 'module=' in cap.text:
# Parse name of the module
module_and_more = cap.text.split('module=')[1]
module_name = module_and_more.split('&')[0]
module_names.append(module_name)
devs = {}
revs = []
# Parse deviations of the module
names = deviations_and_features('deviations=')
# Find and parse deviated yang file so we can get a revision out of it
for i in names:
yang_file = find_first_file('/'.join(self.split[0:-1]), i + '.yang',
i + '.yang')
if yang_file is None:
self.integrity_checker.add_module(self.split, i)
revs.append('1500-01-01')
else:
self.parsed_yang = yangParser.parse(os.path.abspath(yang_file))
yang_variable = self.parsed_yang.search('revision')[0].arg
if isinstance(yang_variable, unicode):
revs.append(unicodedata.normalize('NFKD', yang_variable).encode('ascii', 'ignore'))
else:
revs.append(yang_variable)
if yang_file is not None:
self.integrity_checker.remove_one(self.split, yang_file)
devs['name'] = names
devs['revision'] = revs
deviations[module_name] = devs
# Parse features of the module
features[module_name] = deviations_and_features('features=')
# Parse conformance type of the module
conformance_type[module_name] = 'implement'
# Parse revision of the module from capability.xml file
my_var = ''
if 'revision' in cap.text:
revision_and_more = cap.text.split('revision=')[1]
my_var = revision_and_more.split('&')[0]
revision[module_name] = my_var
else:
yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang',
module_name + '.yang')
self.find_yang_var(revision, 'revision', module_name, yang_file)
name_revision.append(module_name + '@' + revision[module_name])
# Find yang file in the same directory as capability.xml file is
# so we can parse all needed fields out of it
yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang',
module_name + '@' + my_var + '.yang')
if yang_file is None:
# In case we didn`t find the module try to look for it in any other directory of this project
self.integrity_checker.add_module(self.split, module_name)
yang_file = find_first_file('/'.join(self.split[0:1]), module_name + '.yang',
module_name + '@' + my_var + '.yang')
if yang_file is not None:
self.integrity_checker.remove_one(self.split, yang_file)
self.parsed_yang = None
namespace_exist = False
# Parse rest of the fields out of the yang file
self.find_yang_var(namespace, 'namespace', module_name, yang_file)
for ns, org in NS_MAP.items():
if self.os_version is '1651':
if ns is 'urn:cisco':
if ns in namespace[module_name]:
organization_module[module_name] = org
namespace_exist = True
else:
if ns in namespace[module_name]:
organization_module[module_name] = org
namespace_exist = True
if not namespace_exist:
organization_module[module_name] = 'missing_data'
if namespace[module_name] is None:
self.integrity_checker.add_namespace(self.split, module_name + ' : missing data')
namespace[module_name] = 'missing data'
self.integrity_checker.add_namespace(self.split, module_name + ' : ' + namespace[module_name])
self.find_yang_var(prefix, 'prefix', module_name, yang_file)
self.find_yang_var(yang_version, 'yang-version', module_name, yang_file)
self.find_yang_var(organization, 'organization', module_name, yang_file)
self.find_yang_var(contact, 'contact', module_name, yang_file)
self.find_yang_var(description, 'description', module_name, yang_file)
self.find_yang_var(includes, 'include', module_name, yang_file)
self.find_yang_var(imports, 'import', module_name, yang_file)
self.find_yang_var(reference, 'reference', module_name, yang_file)
self.find_yang_var(schema, 'schema', module_name, yang_file)
compilations_status[module_name] = self.parse_status(module_name, revision[module_name])
if compilations_status[module_name] not in 'PASSED':
compilations_result[module_name] = self.parse_result(module_name, revision[module_name])
else:
compilations_result[module_name] = ''
author_email[module_name] = self.parse_email(module_name, revision[module_name])
working_group[module_name] = self.parse_wg(module_name, revision[module_name])
self.prepare.add_key(module_name + '@' + revision[module_name] + '@' + namespace[module_name] + '@'
+ conformance_type[module_name])
self.parse_imports_includes(includes[module_name]['name'], features, revision, name_revision,
yang_version, namespace, prefix, organization, contact, description,
includes, imports, reference, conformance_type, deviations, module_names,
compilations_status, schema, author_email, working_group,
organization_module, True, namespace[module_name], compilations_result)
self.parse_imports_includes(imports[module_name], features, revision, name_revision,
yang_version, namespace, prefix, organization, contact, description,
includes, imports, reference, conformance_type, deviations, module_names,
compilations_status, schema, author_email, working_group,
organization_module, False, namespace[module_name], compilations_result)
# restconf capability parsing
for cap in self.root.iter('module'):
module_name = cap.find('name').text
module_names.append(module_name)
namespace[module_name] = cap.find('namespace').text
revision[module_name] = cap.find('revision').text
schema[module_name] = cap.find('schema').text
conformance_type[module_name] = cap.find('conformance-type').text
devs = {}
names = []
revs = []
for dev in self.root.iter('deviation'):
names.append(dev.find('name').text)
if dev.find('revision').text is None:
revs.append('1500-01-01')
else:
revs.append(dev.find('revision').text)
devs['name'] = names
devs['revision'] = revs
deviations[module_name] = devs
objs = []
for feat in self.root.iter('feature'):
objs.append(feat.text)
features[module_name] = objs
# Find yang file in the same directory as capability.xml file is
# so we can parse all needed fields out of it
yang_file = find_first_file('/'.join(self.split[0:-1]), module_name + '.yang',
module_name + '@' + revision[module_name] + '.yang')
if yang_file is None:
# In case we didn`t find the module try to look for it in any other directory of this project
self.integrity_checker.add_module(self.split, module_name)
yang_file = find_first_file('/'.join(self.split[0:2]), module_name + '.yang',
module_name + '@' + revision[module_name] + '.yang')
# Parse rest of the fields out of the yang file
self.find_yang_var(prefix, 'prefix', module_name, yang_file)
self.find_yang_var(yang_version, 'yang-version', module_name, yang_file)
self.find_yang_var(organization, 'organization', module_name, yang_file)
self.find_yang_var(contact, 'contact', module_name, yang_file)
self.find_yang_var(description, 'description', module_name, yang_file)
self.find_yang_var(reference, 'reference', module_name, yang_file)
self.find_yang_var(includes, 'include', module_name, yang_file)
self.find_yang_var(imports, 'import', module_name, yang_file)
self.integrity_checker.add_unique(name_revision)
# Write dictionary to file
# Create json dictionary out of parsed information
with open('normal' + repr(self.index) + '.json', "w") as ietf_mode:
json.dump({'vendor': self.vendor, 'os-type': self.os, 'os-version': self.os_version,
'platform': self.platform,
'feature-set': 'ALL',
'protocols': {
'protocol': [{
'name': 'netconf',
'capabilities': capability,
'protocol-version': netconf_version,
}]
},
'modules': {
'module': [
{
'reference': reference.get(module_names[k]),
'prefix': prefix.get(module_names[k]),
'yang-version': yang_version.get(module_names[k]),
'organization': organization_module.get(module_names[k]),
'description': description.get(module_names[k]),
'contact': contact.get(module_names[k]),
'submodule': json.loads(
self.get_submodule_info(includes[module_names[k]]['name'])),
# 'imports': json.loads(
# self.get_submodule_info(imports[module_names[k]]['name'], missing_module,
# missing_includes)),
'conformance-type': conformance_type.get(module_names[k]),
'compilation-status': compilations_status.get(module_names[k]),
'author-email': author_email.get(module_names[k]),
'revision': revision.get(module_names[k]),
'namespace': namespace.get(module_names[k]),
'name': module_names[k],
'schema': schema.get(module_names[k]),
'feature': features.get(module_names[k]),
'maturity-level': working_group.get(module_names[k]),
'compilation-result': compilations_result.get(module_names[k]),
'deviation': [
{'name': deviations[module_names[k]]['name'][i],
'revision': deviations[module_names[k]]['revision'][i]
} for
i, val in enumerate(deviations.get(module_names[k])['name'])],
}
for k, val in
enumerate(module_names)]
}
}, ietf_mode)
