def resolve_organization(path):
"""Parse yang file and resolve organization out of the module. If the module
is a submodule find it's parent and resolve its organization
Arguments:
:param path: (str) path to a file to parse and resolve a organization
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
organization = ''
try:
namespace = yangParser.parse(os.path.abspath(path)) \
.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(path)) \
.search('belongs-to')[0].arg
except:
break
try:
yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
try:
yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
organization = MISSING_ELEMENT
return organization
def resolve_organization(path):
"""Parse yang file and resolve organization out of the module. If the module
is a submodule find it's parent and resolve its organization
Arguments:
:param path: (str) path to a file to parse and resolve a organization
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
organization = ''
try:
namespace = yangParser.parse(os.path.abspath(path)) \
.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(path)) \
.search('belongs-to')[0].arg
except:
break
try:
yang_file = find_first_file('/'.join(path.split('/')[:-1]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
try:
yang_file = find_first_file('/'.join(path.split('/')[:-2]), belongs_to + '.yang'
, belongs_to + '@*.yang')
namespace = yangParser.parse(os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = MISSING_ELEMENT
break
except:
organization = MISSING_ELEMENT
return organization
def check_early_revisions(directory):
for f in os.listdir(directory):
fname = f.split('.yang')[0].split('@')[0]
files_to_delete = []
revisions = []
for f2 in os.listdir(directory):
if f2.split('.yang')[0].split('@')[0] == fname:
if f2.split(fname)[1].startswith('.') or f2.split(
fname)[1].startswith('@'):
files_to_delete.append(f2)
revision = f2.split(fname)[1].split('.')[0].replace(
'@', '')
if revision == '':
revision = (yangParser.parse(
os.path.abspath(directory +
f2)).search('revision')[0].arg)
year = int(revision.split('-')[0])
month = int(revision.split('-')[1])
day = int(revision.split('-')[2])
revisions.append(datetime(year, month, day))
latest = revisions.index(max(revisions))
files_to_delete.remove(files_to_delete[latest])
for fi in files_to_delete:
if 'iana-if-type' in fi:
break
os.remove(directory + fi)
def check(self, module_name, path, xml_path):
module_path = Integrity.find_first_file(path, module_name + '.yang', module_name + '@*.yang')
if module_path is None:
self.missing_modules[xml_path].add(module_name)
return
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if module_name not in self.parsed_yang:
self.parsed_yang[module_name] = yangParser.parse(os.path.abspath(module_path), self.ctx)
namespace = None
try:
namespace = self.deserialize(self.parsed_yang[module_name].search('namespace')[0].arg)
except IndexError:
self.missing_wrong_namespaces[xml_path].add(module_name + ': missing_data')
try:
self.deserialize(self.parsed_yang[module_name].search('revision')[0].arg)
except IndexError:
self.missing_revision[xml_path].add(module_name)
try:
imports = self.parsed_yang[module_name].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path, xml_path)
except IndexError:
pass
try:
includes = self.parsed_yang[module_name].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path, xml_path)
except IndexError:
pass
if namespace is not None:
self.check_namespace(path, namespace, module_name, xml_path)
def check_include(self, include, path, xml_path):
module_path = Integrity.find_first_file(path, include + '.yang',
include + '@*.yang')
if module_path is None:
self.missing_submodules[xml_path].add(include)
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if include not in self.parsed_yang:
self.parsed_yang[include] = yangParser.parse(
os.path.abspath(module_path), self.ctx)
try:
self.deserialize(
self.parsed_yang[include].search('revision')[0].arg)
except IndexError:
self.missing_revision[xml_path].add(include)
try:
imports = self.parsed_yang[include].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path, xml_path)
except IndexError:
pass
try:
includes = self.parsed_yang[include].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path,
xml_path)
except IndexError:
pass
def resolve_revision(yang_file):
try:
parsed_yang = yangParser.parse(os.path.abspath(yang_file))
revision = parsed_yang.search('revision')[0].arg
except:
revision = '1970-01-01'
return revision
def get_latest_revision(f):
stmt = yangParser.parse(f)
rev = stmt.search_one('revision')
if rev is None:
return None
return rev.arg
def get_latest_revision(f):
stmt = yangParser.parse(f)
rev = stmt.search_one('revision')
if rev is None:
return None
return rev.arg
def get_specifics(path_dir):
"""Get amount of yang files in specified directory and the amount that
passed compilation
Arguments:
:param path_dir: (str) path to directory where we are searching for
yang files
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
passed = 0
num_in_catalog = 0
for mod_git in list_of_yang_modules_in_subdir(path_dir):
try:
revision = yangParser.parse(
os.path.abspath(mod_git)).search('revision')[0].arg
except:
continue
organization = resolve_organization(mod_git)
name = mod_git.split('/')[-1].split('.')[0].split('@')[0]
if revision is None:
revision = '1500-01-01'
if name is None or organization is None:
continue
if ',' in organization:
organization = organization.replace(' ', '%20')
path = yangcatalog_api_prefix + 'search/name/' + name
module_exist = http_request(path, 'GET', '',
credentials.split(' '),
'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
org = json.loads(
data)['yang-catalog:modules']['module'][0]['organization']
rev = json.loads(
data)['yang-catalog:modules']['module'][0]['revision']
status = json.loads(data)['yang-catalog:modules']['module'][0][
'compilation-status']
if org == organization and rev == revision:
if 'passed' == status:
passed += 1
num_in_catalog += 1
else:
organization = organization.replace(' ', '%20')
path = yangcatalog_api_prefix + 'search/modules/' + name + ','\
+ revision + ',' + organization
module_exist = http_request(path, 'GET', '',
credentials.split(' '),
'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
if 'passed' == json.loads(
data)['module'][0]['compilation-status']:
passed += 1
num_in_catalog += 1
return [num_in_catalog, passed]
def search_dependencies(base):
for root, dirs, files in os.walk(base):
for basename in files:
if fnmatch.fnmatch(basename, '*.yang'):
filename = os.path.join(root, basename)
LOGGER.info('Parsing file {}'.format(filename))
try:
yangFile = yangParser.parse(os.path.abspath(filename))
try:
revision = yangFile.search('revision')[0].arg
except:
revision = '1970-01-01'
name = filename.split('/')[-1].split('.')[0].split('@')[0]
key = '{}@{}.yang'.format(name, revision)
if key not in dependencies:
dependencies[key] = set()
yangImport = yangFile.search('import')
yangInclude = yangFile.search('include')
for imp in yangImport:
impName = imp.arg
impRev = None
for sub in imp.substmts:
if sub.keyword == 'revision-date':
impRev = sub.arg
if impRev:
name_rev = '{}@{}'.format(impName, impRev)
dependencies[key].add(name_rev)
if name_rev not in dependants:
dependants[name_rev] = set()
dependants[name_rev].add(key)
else:
dependencies[key].add(impName)
if impName not in dependants:
dependants[impName] = set()
dependants[impName].add(key)
for inc in yangInclude:
incName = inc.arg
incRev = None
for sub in inc.substmts:
if sub.keyword == 'revision-date':
incRev = sub.arg
if incRev:
name_rev = '{}@{}'.format(incName, incRev)
dependencies[key].add(name_rev)
if name_rev not in dependants:
dependants[name_rev] = set()
dependants[name_rev].add(key)
else:
dependencies[key].add(incName)
if incName not in dependants:
dependants[incName] = set()
dependants[incName].add(key)
except:
pass
def search_dependencies(base):
for root, dirs, files in os.walk(base):
for basename in files:
if fnmatch.fnmatch(basename, '*.yang'):
filename = os.path.join(root, basename)
LOGGER.info('Parsing file {}'.format(filename))
try:
yangFile = yangParser.parse(os.path.abspath(filename))
try:
revision = yangFile.search('revision')[0].arg
except:
revision = '1970-01-01'
name = filename.split('/')[-1].split('.')[0].split('@')[0]
key = '{}@{}.yang'.format(name, revision)
if key not in dependencies:
dependencies[key] = set()
yangImport = yangFile.search('import')
yangInclude = yangFile.search('include')
for imp in yangImport:
impName = imp.arg
impRev = None
for sub in imp.substmts:
if sub.keyword == 'revision-date':
impRev = sub.arg
if impRev:
name_rev = '{}@{}'.format(impName, impRev)
dependencies[key].add(name_rev)
if name_rev not in dependants:
dependants[name_rev] = set()
dependants[name_rev].add(key)
else:
dependencies[key].add(impName)
if impName not in dependants:
dependants[impName] = set()
dependants[impName].add(key)
for inc in yangInclude:
incName = inc.arg
incRev = None
for sub in inc.substmts:
if sub.keyword == 'revision-date':
incRev = sub.arg
if incRev:
name_rev = '{}@{}'.format(incName, incRev)
dependencies[key].add(name_rev)
if name_rev not in dependants:
dependants[name_rev] = set()
dependants[name_rev].add(key)
else:
dependencies[key].add(incName)
if incName not in dependants:
dependants[incName] = set()
dependants[incName].add(key)
except:
pass
def resolve_organization(path):
organization = ''
try:
namespace = yangParser.parse(os.path.abspath(path)) \
.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = yangParser.parse(
os.path.abspath(path)).search('organization')[0].arg
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(path)) \
.search('belongs-to')[0].arg
except:
break
try:
yang_file = find_first_file('/'.join(path.split('/')[:-1]),
belongs_to + '.yang',
belongs_to + '@*.yang')
namespace = yangParser.parse(
os.path.abspath(yang_file)).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
if organization == '':
organization = yangParser.parse(
os.path.abspath(path)).search('organization')[0].arg
break
except:
pass
return organization
def check_name_no_revision_exist(directory):
LOGGER.info('Checking revision for directory: {}'.format(directory))
for root, dirs, files in os.walk(directory):
for basename in files:
if '@' in basename:
yang_file_name = basename.split('@')[0] + '.yang'
revision = basename.split('@')[1].split('.')[0]
exists = os.path.exists(directory + yang_file_name)
if exists:
parsed_yang = yangParser.parse(os.path.abspath(directory + yang_file_name))
comapred_revision = parsed_yang.search('revision')[0].arg
if revision == comapred_revision:
os.remove(directory + yang_file_name)
def check_name_no_revision_exist(directory):
LOGGER.info('Checking revision for directory: {}'.format(directory))
for root, dirs, files in os.walk(directory):
for basename in files:
if '@' in basename:
yang_file_name = basename.split('@')[0] + '.yang'
revision = basename.split('@')[1].split('.')[0]
exists = os.path.exists(directory + yang_file_name)
if exists:
parsed_yang = yangParser.parse(os.path.abspath(directory + yang_file_name))
comapred_revision = parsed_yang.search('revision')[0].arg
if revision == comapred_revision:
os.remove(directory + yang_file_name)
def __resolve_submodule(self):
try:
submodules = self.__parsed_yang.search('include')
except:
return
if len(submodules) == 0:
return
for chunk in submodules:
dep = self.Dependencies()
sub = self.Submodules()
sub.name = chunk.arg
if len(chunk.search('revision-date')) > 0:
sub.revision = chunk.search('revision-date')[0].arg
if sub.revision:
yang_file = self.__find_file(sub.name, sub.revision, True)
dep.revision = sub.revision
else:
yang_file = self.__find_file(sub.name, submodule=True)
try:
sub.revision = \
yangParser.parse(os.path.abspath(yang_file)).search(
'revision')[0].arg
except:
sub.revision = '1970-01-01'
if yang_file is None:
LOGGER.error('Module can not be found')
continue
path = '/'.join(self.schema.split('/')[0:-1])
path += '/{}'.format(yang_file.split('/')[-1])
if yang_file:
sub.schema = path
dep.name = sub.name
dep.schema = sub.schema
self.dependencies.append(dep)
self.submodule.append(sub)
self.json_submodules = json.dumps([{
'name':
self.submodule[x].name,
'schema':
self.submodule[x].schema,
'revision':
self.submodule[x].revision
} for x in range(0, len(self.submodule))])
def get_specifics(path_dir):
"""Get amount of yang files in specified directory and the amount that
passed compilation
Arguments:
:param path_dir: (str) path to directory where we are searching for
yang files
:return: list containing amount of yang files and amount that pass
compilation respectively
"""
passed = 0
num_in_catalog = 0
for mod_git in list_of_yang_modules_in_subdir(path_dir):
try:
revision = yangParser.parse(os.path.abspath(mod_git)).search('revision')[0].arg
except:
continue
organization = resolve_organization(mod_git)
name = mod_git.split('/')[-1].split('.')[0].split('@')[0]
if revision is None:
revision = '1500-01-01'
if name is None or organization is None:
continue
if ',' in organization:
organization = organization.replace(' ', '%20')
path = yangcatalog_api_prefix + 'search/name/' + name
module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
org = json.loads(data)['yang-catalog:modules']['module'][0]['organization']
rev = json.loads(data)['yang-catalog:modules']['module'][0]['revision']
status = json.loads(data)['yang-catalog:modules']['module'][0]['compilation-status']
if org == organization and rev == revision:
if 'passed' == status:
passed += 1
num_in_catalog += 1
else:
organization = organization.replace(' ', '%20')
path = yangcatalog_api_prefix + 'search/modules/' + name + ','\
+ revision + ',' + organization
module_exist = http_request(path, 'GET', '', credentials.split(' '), 'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
if 'passed' == json.loads(data)['module'][0]['compilation-status']:
passed += 1
num_in_catalog += 1
return [num_in_catalog, passed]
def get_specifics(path_dir):
passed = 0
num_in_catalog = 0
for mod_git in list_of_yang_modules_in_subdir(path_dir):
try:
revision = yangParser.parse(
os.path.abspath(mod_git)).search('revision')[0].arg
except:
continue
organization = resolve_organization(mod_git)
name = mod_git.split('/')[-1].split('.')[0].split('@')[0]
if revision is None:
revision = '1500-01-01'
if name is None or organization is None:
continue
if ',' in organization:
path = protocol + '://' + api_ip + ':' + api_port + '/search/name/' + name
module_exist = http_request(path, 'GET', '',
credentials.split(' '),
'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
org = json.loads(
data)['yang-catalog:modules']['module'][0]['organization']
rev = json.loads(
data)['yang-catalog:modules']['module'][0]['revision']
status = json.loads(data)['yang-catalog:modules']['module'][0][
'compilation-status']
if org == organization and rev == revision:
if 'passed' == status:
passed += 1
num_in_catalog += 1
else:
path = protocol + '://' + api_ip + ':' + api_port + '/search/modules/' + name + ',' + revision + ',' + \
organization
module_exist = http_request(path, 'GET', '',
credentials.split(' '),
'application/vnd.yang.data+json')
if module_exist:
data = module_exist.read()
if 'passed' == json.loads(
data)['module'][0]['compilation-status']:
passed += 1
num_in_catalog += 1
return [num_in_catalog, passed]
def add_vendor_information(self, vendor, platform_data, software_version,
os_version, feature_set, os_type,
confarmance_type, capability, netconf_version,
integrity_checker, split):
for data in platform_data:
implementation = self.Implementations()
implementation.vendor = vendor
implementation.platform = data['platform']
implementation.software_version = software_version
implementation.software_flavor = data['software-flavor']
implementation.os_version = os_version
implementation.feature_set = feature_set
implementation.os_type = os_type
implementation.feature = self.features
implementation.capability = capability
implementation.netconf_version = netconf_version
if self.is_yang_lib:
for x in range(0, len(self.deviations['name'])):
devs = implementation.Deviations()
devs.name = self.deviations['name'][x]
devs.revision = self.deviations['revision'][x]
implementation.deviations.append(devs)
else:
for name in self.deviations:
devs = implementation.Deviations()
devs.name = name
yang_file = self.__find_file(name)
if yang_file is None:
devs.revision = '1970-01-01'
else:
try:
s = yang_file.split('/')
key = '/'.join(split[0:-1])
integrity_checker.remove_one(key, s[-1])
devs.revision = yangParser.parse(os.path.abspath(yang_file)) \
.search('revision')[0].arg
except:
devs.revision = '1970-01-01'
implementation.deviations.append(devs)
implementation.conformance_type = confarmance_type
self.implementation.append(implementation)
def find_first_file(directory, pattern, pattern_with_revision):
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern_with_revision):
filename = os.path.join(root, basename)
return filename
for root, dirs, files in os.walk(directory):
for basename in files:
if fnmatch.fnmatch(basename, pattern):
filename = os.path.join(root, basename)
try:
revision = yangParser.parse(filename).search('revision')[0]\
.arg
except:
revision = '1970-01-01'
if '*' not in pattern_with_revision:
if revision in pattern_with_revision:
return filename
else:
return filename
def check(self, module_name, path, xml_path):
module_path = Integrity.find_first_file(path, module_name + '.yang',
module_name + '@*.yang')
if module_path is None:
self.missing_modules[xml_path].add(module_name)
return
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if module_name not in self.parsed_yang:
self.parsed_yang[module_name] = yangParser.parse(
os.path.abspath(module_path), self.ctx)
namespace = None
try:
namespace = self.deserialize(
self.parsed_yang[module_name].search('namespace')[0].arg)
except IndexError:
self.missing_wrong_namespaces[xml_path].add(module_name +
': missing_data')
try:
self.deserialize(
self.parsed_yang[module_name].search('revision')[0].arg)
except IndexError:
self.missing_revision[xml_path].add(module_name)
try:
imports = self.parsed_yang[module_name].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path, xml_path)
except IndexError:
pass
try:
includes = self.parsed_yang[module_name].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path,
xml_path)
except IndexError:
pass
if namespace is not None:
self.check_namespace(path, namespace, module_name, xml_path)
def __resolve_submodule_case(self, field):
if self.module_type == 'submodule':
LOGGER.debug(
'Getting parent information because file {} is a submodule'.
format(self.__path))
yang_file = self.__find_file(self.belongs_to)
if yang_file is None:
return None
parsed_parent_yang = yangParser.parse(os.path.abspath(yang_file))
try:
return parsed_parent_yang.search(field)[0].arg
except:
if field == 'prefix':
return None
else:
return MISSING_ELEMENT
else:
try:
return self.__parsed_yang.search(field)[0].arg
except:
if field == 'prefix':
return None
else:
return MISSING_ELEMENT
def check_early_revisions(directory):
for f in os.listdir(directory):
fname = f.split('.yang')[0].split('@')[0]
files_to_delete = []
revisions = []
for f2 in os.listdir(directory):
if f2.split('.yang')[0].split('@')[0] == fname:
if f2.split(fname)[1].startswith('.') or f2.split(fname)[1].startswith('@'):
files_to_delete.append(f2)
revision = f2.split(fname)[1].split('.')[0].replace('@', '')
if revision == '':
revision = (yangParser.parse(os.path.abspath(directory
+ f2))
.search('revision')[0].arg)
year = int(revision.split('-')[0])
month = int(revision.split('-')[1])
day = int(revision.split('-')[2])
revisions.append(datetime(year, month, day))
latest = revisions.index(max(revisions))
files_to_delete.remove(files_to_delete[latest])
for fi in files_to_delete:
if 'iana-if-type' in fi:
break
os.remove(directory + fi)
def check_include(self, include, path, xml_path):
module_path = Integrity.find_first_file(path, include + '.yang', include + '@*.yang')
if module_path is None:
self.missing_submodules[xml_path].add(include)
if module_path in self.useless_modules:
self.useless_modules.remove(module_path)
if include not in self.parsed_yang:
self.parsed_yang[include] = yangParser.parse(os.path.abspath(module_path), self.ctx)
try:
self.deserialize(self.parsed_yang[include].search('revision')[0].arg)
except IndexError:
self.missing_revision[xml_path].add(include)
try:
imports = self.parsed_yang[include].search('import')
for imp in imports:
self.check(self.deserialize(imp.arg), path, xml_path)
except IndexError:
pass
try:
includes = self.parsed_yang[include].search('include')
for include in includes:
self.check_include(self.deserialize(include.arg), path, xml_path)
except IndexError:
pass
def add_modules():
if not request.json:
abort(400)
body = request.json
LOGGER.info('Adding modules with body {}'.format(body))
tree_created = False
with open('./prepare-sdo.json', "w") as plat:
json.dump(body, plat)
try:
http_request(
protocol + '://' + confd_ip + ':' + repr(confdPort) +
'/api/config/modules/', 'DELETE', None, credentials,
'application/vnd.yang.collection+json')
except:
pass
path = protocol + '://' + confd_ip + ':' + repr(
confdPort) + '/api/config/modules'
try:
http_request(path, 'PUT', json.dumps(body), credentials,
'application/vnd.yang.data+json')
except urllib2.HTTPError as e:
abort(e.code)
repo = {}
warning = []
direc = 0
while True:
if os.path.isdir('../parseAndPopulate/' + repr(direc)):
direc += 1
else:
break
direc = '../parseAndPopulate/' + repr(direc)
for mod in body['modules']['module']:
sdo = mod['source-file']
orgz = mod['organization']
if request.method == 'POST':
try:
path = protocol + '://' + confd_ip + ':' + repr(confdPort) + '/api/config/catalog/modules/module/' + \
mod['name'] + ',' + mod['revision'] + ',' + mod['organization']
http_request(path, 'GET', None, credentials,
'application/vnd.yang.data+json')
continue
except urllib2.HTTPError as e:
pass
directory = '/'.join(sdo['path'].split('/')[:-1])
repo_url = url + sdo['owner'] + '/' + sdo['repository']
LOGGER.debug('Cloning repository')
if repo_url not in repo:
repo[repo_url] = repoutil.RepoUtil(repo_url)
repo[repo_url].clone()
for submodule in repo[repo_url].repo.submodules:
submodule.update(init=True)
if sdo.get('branch'):
branch = sdo.get('branch')
else:
branch = 'master'
save_to = direc + '/temp/' + sdo['owner'] + '/' + sdo['repository'].split('.')[0] \
+ '/' + branch + '/' + directory
try:
os.makedirs(save_to)
except OSError as e:
# be happy if someone already created the path
if e.errno != errno.EEXIST:
raise
shutil.copy(repo[repo_url].localdir + '/' + sdo['path'], save_to)
if os.path.isfile('./prepare-sdo.json'):
shutil.move('./prepare-sdo.json', direc)
tree_created = True
organization = ''
try:
namespace = yangParser.parse(os.path.abspath(save_to + '/' + sdo['path'].split('/')[-1])) \
.search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(':')[0]
except:
while True:
try:
belongs_to = yangParser.parse(os.path.abspath(repo[repo_url].localdir + '/' + sdo['path'])) \
.search('belongs-to')[0].arg
except:
break
try:
namespace = yangParser.parse(
os.path.abspath(repo[repo_url].localdir + '/' +
'/'.join(sdo['path'].split('/')[:-1]) +
'/' + belongs_to +
'.yang')).search('namespace')[0].arg
for ns, org in NS_MAP.items():
if ns in namespace:
organization = org
if organization == '':
if 'urn:' in namespace:
organization = namespace.split('urn:')[1].split(
':')[0]
break
except:
pass
resolved_authorization = authorize_for_sdos(request, orgz,
organization)
if not resolved_authorization:
shutil.rmtree(direc)
for key in repo:
repo[key].remove()
return unauthorized()
if 'organization' in repr(resolved_authorization):
warning.append(sdo['path'].split('/')[-1] + ' ' +
resolved_authorization)
for key in repo:
repo[key].remove()
api_protocol = 'http'
api_port = 5000
if ssl_key != '':
api_protocol = 'https'
api_port = 8443
LOGGER.debug('Sending a new job')
arguments = [
"python", "../parseAndPopulate/populate.py", "--sdo", "--port",
repr(confdPort), "--dir", direc + "/temp", "--api", "--ip", confd_ip,
"--credentials", credentials[0], credentials[1],
repr(tree_created), protocol, api_protocol,
repr(api_port)
]
job_id = sender.send('#'.join(arguments))
LOGGER.info('job_id {}'.format(job_id))
if len(warning) > 0:
return jsonify({
'info': 'Verification successful',
'job-id': job_id,
'warnings': [{
'warning': val
} for val in warning]
})
else:
return jsonify({'info': 'Verification successful', 'job-id': job_id})
def __init__(self,
path,
html_result_dir,
jsons,
temp_dir,
is_vendor=False,
is_yang_lib=False,
data=None,
is_vendor_imp_inc=False,
run_integrity=False):
self.run_integrity = run_integrity
self.__temp_dir = temp_dir
self.__missing_submodules = []
self.__missing_modules = []
self.__missing_namespace = None
self.__missing_revision = None
self.is_yang_lib = is_yang_lib
self.html_result_dir = html_result_dir
self.jsons = jsons
self.__is_vendor = is_vendor
self.revision = '*'
self.__path = path
self.features = []
self.deviations = []
if is_vendor:
if is_yang_lib:
self.deviations = data['deviations']
self.features = data['features']
self.revision = data['revision']
if self.revision is None:
self.revision = '*'
self.__path = self.__find_file(data['name'], self.revision)
else:
self.features = self.\
__resolve_deviations_and_features('features=', data)
self.deviations = \
self.__resolve_deviations_and_features('deviations=', data)
if 'revision' in data:
revision_and_more = data.split('revision=')[1]
revision = revision_and_more.split('&')[0]
self.revision = revision
self.__path = self.__find_file(
data.split('&')[0], self.revision)
else:
self.__path = path
if is_vendor_imp_inc:
self.__is_vendor = True
if self.__path:
self.name = None
self.organization = None
self.ietf_wg = None
self.namespace = None
self.schema = None
self.generated_from = None
self.maturity_level = None
self.document_name = None
self.author_email = None
self.reference = None
self.tree = None
self.expired = None
self.expiration_date = None
self.module_classification = None
self.compilation_status = None
self.compilation_result = {}
self.prefix = None
self.yang_version = None
self.description = None
self.contact = None
self.belongs_to = None
self.submodule = []
self.dependencies = []
self.module_type = None
self.tree_type = None
self.semver = None
self.derived_semver = None
self.implementation = []
self.imports = []
self.json_submodules = json.dumps([])
self.__parsed_yang = yangParser.parse(os.path.abspath(self.__path))
if self.__parsed_yang is None:
pass
# TODO file has wrong format. probably end with CODE END
else:
raise FileError(path.split('&')[0], 'File not found')