def create_tree(name: str, revision: str):
"""
Return yang tree representation of yang module with corresponding module name and revision.
Arguments:
:param name (str) name of the module
:param revision (str) revision of the module in format YYYY-MM-DD
:return preformatted HTML with corresponding data
"""
path_to_yang = '{}/{}@{}.yang'.format(ac.d_save_file_dir, name, revision)
plugin.plugins = []
plugin.init([])
ctx = create_context('{}:{}'.format(ac.d_yang_models_dir,
ac.d_save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
try:
with open(path_to_yang, 'r') as f:
a = ctx.add_module(path_to_yang, f.read())
except:
abort(400, description='File {} was not found'.format(path_to_yang))
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
ctx.validate()
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth, ctx.opts.tree_line_length,
path)
stdout = f.getvalue()
if stdout == '' and len(ctx.errors) != 0:
message = 'This yang file contains major errors and therefore tree can not be created.'
return create_bootstrap_danger(message)
elif stdout != '' and len(ctx.errors) != 0:
message = 'This yang file contains some errors, but tree was created.'
return create_bootstrap_warning(stdout, message)
elif stdout == '' and len(ctx.errors) == 0:
return create_bootstrap_info()
else:
return '<html><body><pre>{}</pre></body></html>'.format(stdout)
def context_check_update_from(old_schema: str, new_schema: str, yang_models: str, save_file_dir: str):
""" Perform pyang --check-update-from validation using context.
Argumets:
:param old_schema (str) full path to the yang file with older revision
:param new_schema (str) full path to the yang file with newer revision
:param yang_models (str) path to the directory where YangModels/yang repo is cloned
:param save_file_dir (str) path to the directory where all the yang files will be saved
"""
plugin.plugins = []
plugin.init([])
ctx = create_context(
'{}:{}'.format(os.path.abspath(yang_models), save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
optParser = optparse.OptionParser('', add_help_option=False)
for p in plugin.plugins:
p.setup_ctx(ctx)
p.add_opts(optParser)
with open(new_schema, 'r', errors='ignore') as f:
new_schema_ctx = ctx.add_module(new_schema, f.read())
ctx.opts.check_update_from = old_schema
ctx.opts.old_path = [os.path.abspath(yang_models)]
ctx.opts.verbose = False
ctx.opts.old_deviation = []
retry = 5
while retry:
try:
ctx.validate()
# NOTE: ResourceWarning appears due to the incorrect way pyang opens files for reading
# ResourceWarning: Enable tracemalloc to get the object allocation traceback
with warnings.catch_warnings(record=True):
check_update(ctx, new_schema_ctx)
break
except Exception as e:
retry -= 1
if retry == 0:
raise e
return ctx, new_schema_ctx
def is_transitional(rows, output):
if output.split('\n')[1].endswith('-state') and output.split(
'\n')[0].endswith('-state'):
if '+--rw' in output:
return False
if output.startswith('\n'):
name_of_module = output.split('\n')[1].split(': ')[1]
else:
name_of_module = output.split('\n')[0].split(': ')[1]
name_of_module = name_of_module.split('-state')[0]
coresponding_nmda_file = self._find_file(name_of_module)
if coresponding_nmda_file:
name = coresponding_nmda_file.split('/')[-1].split('.')[0]
revision = name.split('@')[-1]
name = name.split('@')[0]
if '{}@{}'.format(name, revision) in self._trees:
stdout = self._trees[name][revision]
pyang_list_of_rows = stdout.split('\n')[2:]
else:
plugin.plugins = []
plugin.init([])
ctx = create_context('{}:{}'.format(
os.path.abspath(self._yang_models),
self._save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(coresponding_nmda_file, 'r') as f:
a = ctx.add_module(coresponding_nmda_file,
f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
ctx.validate()
try:
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
stdout = f.getvalue()
except:
stdout = ''
pyang_list_of_rows = stdout.split('\n')[2:]
if len(ctx.errors) != 0 and len(stdout) == 0:
return False
if stdout == '':
return False
for x in range(0, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '':
break
if len(rows[x].split('+--')[0]) == 4:
if '-state' in rows[x]:
return False
if len(rows[x].split('augment')[0]) == 2:
part = rows[x].strip(' ').split('/')[1]
if '-state' in part:
return False
if '+--ro ' in rows[x]:
leaf = \
rows[x].split('+--ro ')[1].split(' ')[0].split(
'?')[0]
for y in range(0, len(pyang_list_of_rows)):
if leaf in pyang_list_of_rows[y]:
break
else:
return False
return True
else:
return False
else:
return False
def resolve_tree_type(self, all_modules):
def is_openconfig(rows, output):
count_config = output.count('+-- config')
count_state = output.count('+-- state')
if count_config != count_state:
return False
row_number = 0
skip = []
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if '' == row.strip(' '):
break
if '+--rw' in row and row_number != 0 \
and row_number not in skip and '[' not in row and \
(len(row.replace('|', '').strip(' ').split(
' ')) != 2 or '(' in row):
if '->' in row and 'config' in row.split('->')[
1] and '+--rw config' not in rows[row_number - 1]:
row_number += 1
continue
if '+--rw config' not in rows[row_number - 1]:
if 'augment' in rows[row_number - 1]:
if not rows[row_number - 1].endswith(':config:'):
return False
else:
return False
length_before = set([len(row.split('+--')[0])])
skip = []
for x in range(row_number, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if len(rows[x].split('+--')[0]) not in length_before:
if (len(rows[x].replace('|', '').strip(' ').split(
' ')) != 2 and '[' not in rows[x]) \
or '+--:' in rows[x] or '(' in rows[x]:
length_before.add(len(rows[x].split('+--')[0]))
else:
break
if '+--ro' in rows[x]:
return False
duplicate = \
rows[x].replace('+--rw', '+--ro').split('+--')[1]
if duplicate.replace(' ', '') not in output.replace(
' ', ''):
return False
skip.append(x)
if '+--ro' in row and row_number != 0 and row_number not in skip and '[' not in row and \
(len(row.replace('|', '').strip(' ').split(
' ')) != 2 or '(' in row):
if '->' in row and 'state' in row.split(
'->')[1] and '+--ro state' not in rows[row_number -
1]:
row_number += 1
continue
if '+--ro state' not in rows[row_number - 1]:
if 'augment' in rows[row_number - 1]:
if not rows[row_number - 1].endswith(':state:'):
return False
else:
return False
length_before = len(row.split('+--')[0])
skip = []
for x in range(row_number, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if len(rows[x].split('+--')[0]) < length_before:
break
if '+--rw' in rows[x]:
return False
skip.append(x)
row_number += 1
return True
def is_combined(rows, output):
for row in rows:
if row.endswith('-state') and not ('x--' in row
or 'o--' in row):
return False
next_obsolete_or_deprecated = False
for row in rows:
if next_obsolete_or_deprecated:
if 'x--' in row or 'o--' in row:
next_obsolete_or_deprecated = False
else:
return False
if 'x--' in row or 'o--' in row:
continue
if '+--rw config' == row.replace(
'|', '').strip(' ') or '+--ro state' == row.replace(
'|', '').strip(' '):
return False
if len(row.split('+--')[0]) == 4:
if '-state' in row and '+--ro' in row:
return False
if 'augment' in row and len(row.split('augment')[0]) == 2:
part = row.strip(' ').split('/')[1]
if '-state' in part:
next_obsolete_or_deprecated = True
part = row.strip(' ').split('/')[-1]
if ':state:' in part or '/state:' in part \
or ':config:' in part or '/config:' in part:
next_obsolete_or_deprecated = True
return True
def is_transitional(rows, output):
if output.split('\n')[1].endswith('-state') and output.split(
'\n')[0].endswith('-state'):
if '+--rw' in output:
return False
if output.startswith('\n'):
name_of_module = output.split('\n')[1].split(': ')[1]
else:
name_of_module = output.split('\n')[0].split(': ')[1]
name_of_module = name_of_module.split('-state')[0]
coresponding_nmda_file = self._find_file(name_of_module)
if coresponding_nmda_file:
name = coresponding_nmda_file.split('/')[-1].split('.')[0]
revision = name.split('@')[-1]
name = name.split('@')[0]
if '{}@{}'.format(name, revision) in self._trees:
stdout = self._trees[name][revision]
pyang_list_of_rows = stdout.split('\n')[2:]
else:
plugin.plugins = []
plugin.init([])
ctx = create_context('{}:{}'.format(
os.path.abspath(self._yang_models),
self._save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(coresponding_nmda_file, 'r') as f:
a = ctx.add_module(coresponding_nmda_file,
f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
ctx.validate()
try:
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
stdout = f.getvalue()
except:
stdout = ''
pyang_list_of_rows = stdout.split('\n')[2:]
if len(ctx.errors) != 0 and len(stdout) == 0:
return False
if stdout == '':
return False
for x in range(0, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '':
break
if len(rows[x].split('+--')[0]) == 4:
if '-state' in rows[x]:
return False
if len(rows[x].split('augment')[0]) == 2:
part = rows[x].strip(' ').split('/')[1]
if '-state' in part:
return False
if '+--ro ' in rows[x]:
leaf = \
rows[x].split('+--ro ')[1].split(' ')[0].split(
'?')[0]
for y in range(0, len(pyang_list_of_rows)):
if leaf in pyang_list_of_rows[y]:
break
else:
return False
return True
else:
return False
else:
return False
def is_split(rows, output):
failed = False
row_num = 0
if output.split('\n')[1].endswith('-state'):
return False
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if '+--rw config' == row.replace('|', '').strip(
' ') or '+--ro state' == row.replace('|', '') \
.strip(' '):
return False
if 'augment' in row:
part = row.strip(' ').split('/')[-1]
if ':state:' in part or '/state:' in part or ':config:' in part or '/config:' in part:
return False
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if row == '':
break
if (len(row.split('+--')[0]) == 4
and 'augment' not in rows[row_num - 1]) or len(
row.split('augment')[0]) == 2:
if '-state' in row:
if 'augment' in row:
part = row.strip(' ').split('/')[1]
if '-state' not in part:
row_num += 1
continue
for x in range(row_num + 1, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '' \
or (len(rows[x].split('+--')[
0]) == 4 and 'augment' not in
rows[row_num - 1]) \
or len(row.split('augment')[0]) == 2:
break
if '+--rw' in rows[x]:
failed = True
break
row_num += 1
if failed:
return False
else:
return True
for x, module in enumerate(all_modules.get('module', []), start=1):
name = module['name']
revision = module['revision']
name_revision = '{}@{}'.format(name, revision)
self._path = '{}/{}.yang'.format(self._save_file_dir,
name_revision)
yang_file_exists = self._check_schema_file(module)
is_latest_revision = self.check_if_latest_revision(module)
if not yang_file_exists:
LOGGER.error('Skipping module: {}'.format(name_revision))
continue
LOGGER.info('Searching tree-type for {}. {} out of {}'.format(
name_revision, x, len(all_modules['module'])))
if revision in self._trees[name]:
stdout = self._trees[name][revision]
else:
plugin.plugins = []
plugin.init([])
ctx = create_context('{}:{}'.format(
os.path.abspath(self._yang_models), self._save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(self._path, 'r', errors='ignore') as f:
a = ctx.add_module(self._path, f.read())
if a is None:
LOGGER.debug(
'Could not use pyang to generate tree because of errors on module {}'
.format(self._path))
module['tree-type'] = 'unclassified'
if revision not in self.new_modules[name]:
self.new_modules[name][revision] = module
else:
self.new_modules[name][revision][
'tree-type'] = 'unclassified'
continue
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
retry = 5
while retry:
try:
ctx.validate()
break
except Exception as e:
retry -= 1
if retry == 0:
raise e
try:
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
stdout = f.getvalue()
self._trees[name][revision] = stdout
except:
module['tree-type'] = 'not-applicable'
LOGGER.exception('not-applicable tree created')
continue
if stdout == '':
module['tree-type'] = 'not-applicable'
else:
if stdout.startswith('\n'):
pyang_list_of_rows = stdout.split('\n')[2:]
else:
pyang_list_of_rows = stdout.split('\n')[1:]
if 'submodule' == module['module-type']:
LOGGER.debug('Module {} is a submodule'.format(self._path))
module['tree-type'] = 'not-applicable'
elif is_latest_revision and is_combined(
pyang_list_of_rows, stdout):
module['tree-type'] = 'nmda-compatible'
elif is_split(pyang_list_of_rows, stdout):
module['tree-type'] = 'split'
elif is_openconfig(pyang_list_of_rows, stdout):
module['tree-type'] = 'openconfig'
elif is_transitional(pyang_list_of_rows, stdout):
module['tree-type'] = 'transitional-extra'
else:
module['tree-type'] = 'unclassified'
LOGGER.debug('tree type for module {} is {}'.format(
module['name'], module['tree-type']))
if (revision not in self._existing_modules_dict[name]
or self._existing_modules_dict[name][revision].get(
'tree-type') != module['tree-type']):
LOGGER.info('tree-type {} vs {} for module {}@{}'.format(
self._existing_modules_dict[name].get(revision,
{}).get('tree-type'),
module['tree-type'], module['name'], module['revision']))
if revision not in self.new_modules[name]:
self.new_modules[name][revision] = module
else:
self.new_modules[name][revision]['tree-type'] = module[
'tree-type']
def tree_module_revision(module_name, revision):
"""
Generates yang tree view of the module.
:param module_name: Module for which we are generating the tree.
:param revision : Revision of the module
:return: json response with yang tree
"""
response = {}
alerts = []
jstree_json = {}
nmodule = os.path.basename(module_name)
if nmodule != module_name:
abort(400, description='Invalid module name specified')
else:
revisions, organization = get_modules_revision_organization(module_name, revision)
if len(revisions) == 0:
abort(404, description='Provided module does not exist')
if revision is None:
# get latest revision of provided module
revision = revisions[0]
path_to_yang = '{}/{}@{}.yang'.format(ac.d_save_file_dir, module_name, revision)
plugin.plugins = []
plugin.init([])
ctx = create_context('{}'.format(ac.d_yang_models_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
try:
with open(path_to_yang, 'r') as f:
module_context = ctx.add_module(path_to_yang, f.read())
assert module_context
except Exception:
msg = 'File {} was not found'.format(path_to_yang)
bp.LOGGER.exception(msg)
abort(400, description=msg)
imports_includes = []
imports_includes.extend(module_context.search('import'))
imports_includes.extend(module_context.search('include'))
import_include_map = {}
for imp_inc in imports_includes:
prefix = imp_inc.search('prefix')
if len(prefix) == 1:
prefix = prefix[0].arg
else:
prefix = 'None'
import_include_map[prefix] = imp_inc.arg
json_ytree = ac.d_json_ytree
yang_tree_file_path = '{}/{}@{}.json'.format(json_ytree, module_name, revision)
response['maturity'] = get_module_data('{}@{}/{}'.format(module_name, revision,
organization)).get('maturity-level', '').upper()
response['import-include'] = import_include_map
if os.path.isfile(yang_tree_file_path):
try:
with open(yang_tree_file_path) as f:
json_tree = json.load(f)
if json_tree is None:
alerts.append('Failed to decode JSON data: ')
else:
response['namespace'] = json_tree.get('namespace', '')
response['prefix'] = json_tree.get('prefix', '')
import_include_map[response['prefix']] = module_name
data_nodes = build_tree(json_tree, module_name, import_include_map)
jstree_json = dict()
jstree_json['data'] = [data_nodes]
if json_tree.get('rpcs') is not None:
rpcs = dict()
rpcs['name'] = json_tree['prefix'] + ':rpcs'
rpcs['children'] = json_tree['rpcs']
jstree_json['data'].append(build_tree(rpcs, module_name, import_include_map))
if json_tree.get('notifications') is not None:
notifs = dict()
notifs['name'] = json_tree['prefix'] + ':notifs'
notifs['children'] = json_tree['notifications']
jstree_json['data'].append(build_tree(notifs, module_name, import_include_map))
if json_tree.get('augments') is not None:
augments = dict()
augments['name'] = json_tree['prefix'] + ':augments'
augments['children'] = []
for aug in json_tree.get('augments'):
aug_info = dict()
aug_info['name'] = aug['augment_path']
aug_info['children'] = aug['augment_children']
augments['children'].append(aug_info)
jstree_json['data'].append(build_tree(augments, module_name, import_include_map, augments=True))
except Exception as e:
alerts.append("Failed to read YANG tree data for {}@{}/{}, {}".format(module_name, revision,
organization, e))
else:
alerts.append('YANG Tree data does not exist for {}@{}/{}'.format(module_name, revision, organization))
if jstree_json is None:
response['jstree_json'] = dict()
alerts.append('Json tree could not be generated')
else:
response['jstree_json'] = jstree_json
response['module'] = '{}@{}'.format(module_name, revision)
response['warning'] = alerts
return make_response(jsonify(response), 200)
args = parser.parse_args()
config_path = args.config_path
config = create_config(config_path)
save_file_dir = config.get('Directory-Section', 'save-file-dir')
json_ytree = config.get('Directory-Section', 'json-ytree')
jsons = glob.glob('{}/*.json'.format(json_ytree))
num_of_jsons = len(jsons)
i = 0
for i, jsn in enumerate(jsons):
print('tree {} {} out of {}'.format(jsn, i + 1, num_of_jsons))
file_stat = Path(jsn).stat()
if file_stat.st_size != 0:
continue
plugin.init([])
ctx = create_context(save_file_dir)
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
for p in plugin.plugins:
p.setup_ctx(ctx)
module = jsn.split('/')[-1]
name_revision = module.split('@')
name = name_revision[0]
revision = name_revision[1].split('.')[0]
all_modules_path = '{}/{}@{}.yang'.format(save_file_dir, name,
revision)
parsed_module = None
try:
with open(all_modules_path, 'r') as f:
parsed_module = ctx.add_module(all_modules_path, f.read())
except Exception:
def __parse_semver(self, existing_modules):
z = 0
for module in self.__all_modules['module']:
z += 1
data = {}
for m in existing_modules:
if m['name'] == module['name']:
if m['revision'] != module['revision']:
data['{}@{}'.format(m['name'], m['revision'])] = m
LOGGER.info('Searching semver for {}. {} out of {}'.format(
module['name'], z, len(self.__all_modules['module'])))
if len(data) == 0:
module['derived-semantic-version'] = '1.0.0'
self.__new_modules.append(module)
else:
rev = module['revision'].split('-')
try:
date = datetime(int(rev[0]), int(rev[1]), int(rev[2]))
except Exception as e:
LOGGER.error(
'Failed to process revision for {}: (rev: {})'.format(
module['name'], rev))
try:
if int(rev[1]) == 2 and int(rev[2]) == 29:
date = datetime(int(rev[0]), int(rev[1]), 28)
else:
date = datetime(1970, 1, 1)
except Exception:
date = datetime(1970, 1, 1)
module_temp = {}
module_temp['name'] = module['name']
module_temp['revision'] = module['revision']
module_temp['organization'] = module['organization']
module_temp['compilation'] = module['compilation-status']
module_temp['date'] = date
module_temp['schema'] = module['schema']
modules = [module_temp]
semver_exist = True
if sys.version_info >= (3, 4):
mod_list = data.items()
else:
mod_list = data.iteritems()
for key, mod in mod_list:
module_temp = {}
revision = mod['revision']
if revision == module['revision']:
continue
rev = revision.split('-')
module_temp['revision'] = revision
try:
module_temp['date'] = datetime(int(rev[0]),
int(rev[1]),
int(rev[2]))
except Exception:
LOGGER.warning(
'Failed to process revision for {}: (rev: {}) setting it to 28th'
.format(module['name'], rev))
try:
if int(rev[1]) == 2 and int(rev[2]) == 29:
module_temp['date'] = datetime(
int(rev[0]), int(rev[1]), 28)
else:
module_temp['date'] = datetime(1970, 1, 1)
except Exception:
module_temp['date'] = datetime(1970, 1, 1)
module_temp['name'] = mod['name']
module_temp['organization'] = mod.get('organization')
module_temp['schema'] = mod.get('schema')
module_temp['compilation'] = mod.get('compilation-status')
module_temp['semver'] = mod.get('derived-semantic-version')
if module_temp['semver'] is None:
semver_exist = False
modules.append(module_temp)
data['{}@{}'.format(module['name'],
module['revision'])] = module
if len(modules) == 1:
module['derived-semantic-version'] = '1.0.0'
self.__new_modules.append(module)
continue
modules = sorted(modules, key=lambda k: k['date'])
# If we are adding new module to the end (latest revision) of existing modules with this name
# and all modules with this name have semver already assigned exept the last one
if modules[-1]['date'] == date and semver_exist:
if modules[-1]['compilation'] != 'passed':
versions = modules[-2]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(ver, 0, 0)
module['derived-semantic-version'] = upgraded_version
self.__new_modules.append(module)
else:
if modules[-2]['compilation'] != 'passed':
versions = modules[-2]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(ver, 0, 0)
module[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(module)
continue
else:
schema2 = '{}/{}@{}.yang'.format(
self.__save_file_dir, modules[-2]['name'],
modules[-2]['revision'])
schema1 = '{}/{}@{}.yang'.format(
self.__save_file_dir, modules[-1]['name'],
modules[-1]['revision'])
ctx = create_context('{}:{}'.format(
os.path.abspath(self.__yang_models),
self.__save_file_dir))
with open(schema1, 'r', errors='ignore') as f:
a1 = ctx.add_module(schema1, f.read())
ctx.opts.check_update_from = schema2
ctx.opts.old_path = [
os.path.abspath(self.__yang_models)
]
ctx.opts.verbose = False
ctx.opts.old_path = []
ctx.opts.old_deviation = []
check_update(ctx, schema2, a1)
if len(ctx.errors) == 0:
with open(schema2, 'r', errors='ignore') as f:
a2 = ctx.add_module(schema2, f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
try:
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'w') as f:
emit_tree(ctx, [a1], f,
ctx.opts.tree_depth,
ctx.opts.tree_line_length,
path)
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'r') as f:
stdout = f.read()
except:
stdout = ""
try:
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'w') as f:
emit_tree(ctx, [a2], f,
ctx.opts.tree_depth,
ctx.opts.tree_line_length,
path)
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'r') as f:
stdout2 = f.read()
except:
stdout2 = "2"
os.unlink('{}/pyang_temp.txt'.format(
self.temp_dir))
if stdout == stdout2:
versions = modules[-2]['semver'].split('.')
ver = int(versions[2])
ver += 1
upgraded_version = '{}.{}.{}'.format(
versions[0], versions[1], ver)
module[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(module)
continue
else:
versions = modules[-2]['semver'].split('.')
ver = int(versions[1])
ver += 1
upgraded_version = '{}.{}.{}'.format(
versions[0], ver, 0)
module[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(module)
continue
else:
versions = modules[-2]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(ver, 0, 0)
module[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(module)
continue
else:
mod = {}
mod['name'] = modules[0]['name']
mod['revision'] = modules[0]['revision']
mod['organization'] = modules[0]['organization']
modules[0]['semver'] = '1.0.0'
response = data['{}@{}'.format(mod['name'],
mod['revision'])]
response['derived-semantic-version'] = '1.0.0'
self.__new_modules.append(response)
for x in range(1, len(modules)):
mod = {}
mod['name'] = modules[x]['name']
mod['revision'] = modules[x]['revision']
mod['organization'] = modules[x]['organization']
if modules[x]['compilation'] != 'passed':
versions = modules[x - 1]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(ver, 0, 0)
modules[x]['semver'] = upgraded_version
response = data['{}@{}'.format(
mod['name'], mod['revision'])]
response[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(response)
else:
if modules[x - 1]['compilation'] != 'passed':
versions = modules[x - 1]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(ver, 0, 0)
modules[x]['semver'] = upgraded_version
response = data['{}@{}'.format(
mod['name'], mod['revision'])]
response[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(response)
continue
else:
schema2 = '{}/{}@{}.yang'.format(
self.__save_file_dir, modules[x]['name'],
modules[x]['revision'])
schema1 = '{}/{}@{}.yang'.format(
self.__save_file_dir,
modules[x - 1]['name'],
modules[x - 1]['revision'])
ctx = create_context('{}:{}'.format(
os.path.abspath(self.__yang_models),
self.__save_file_dir))
with open(schema1, 'r', errors='ignore') as f:
a1 = ctx.add_module(schema1, f.read())
ctx.opts.check_update_from = schema2
ctx.opts.old_path = [
os.path.abspath(self.__yang_models)
]
ctx.opts.verbose = False
ctx.opts.old_path = []
ctx.opts.old_deviation = []
check_update(ctx, schema2, a1)
if len(ctx.errors) == 0:
with open(schema2, 'r',
errors='ignore') as f:
a2 = ctx.add_module(schema2, f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
try:
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'w') as f:
emit_tree(
ctx, [a1], f,
ctx.opts.tree_depth,
ctx.opts.tree_line_length,
path)
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'r') as f:
stdout = f.read()
except:
stdout = ""
try:
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'w') as f:
emit_tree(
ctx, [a2], f,
ctx.opts.tree_depth,
ctx.opts.tree_line_length,
path)
with open(
'{}/pyang_temp.txt'.format(
self.temp_dir), 'r') as f:
stdout2 = f.read()
except:
stdout2 = "2"
os.unlink('{}/pyang_temp.txt'.format(
self.temp_dir))
if stdout == stdout2:
versions = modules[
x - 1]['semver'].split('.')
ver = int(versions[2])
ver += 1
upgraded_version = '{}.{}.{}'.format(
versions[0], versions[1], ver)
modules[x]['semver'] = upgraded_version
response = data['{}@{}'.format(
mod['name'], mod['revision'])]
response[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(response)
else:
versions = modules[
x - 1]['semver'].split('.')
ver = int(versions[1])
ver += 1
upgraded_version = '{}.{}.{}'.format(
versions[0], ver, 0)
modules[x]['semver'] = upgraded_version
response = data['{}@{}'.format(
mod['name'], mod['revision'])]
response[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(response)
else:
versions = modules[x -
1]['semver'].split('.')
ver = int(versions[0])
ver += 1
upgraded_version = '{}.{}.{}'.format(
ver, 0, 0)
modules[x]['semver'] = upgraded_version
response = data['{}@{}'.format(
mod['name'], mod['revision'])]
response[
'derived-semantic-version'] = upgraded_version
self.__new_modules.append(response)
def is_transational(rows, output):
if output.split('\n')[1].endswith('-state'):
if '+--rw' in output:
return False
if output.startswith('\n'):
name_of_module = output.split('\n')[1].split(': ')[1]
else:
name_of_module = output.split('\n')[0].split(': ')[1]
name_of_module = name_of_module.split('-state')[0]
coresponding_nmda_file = self.__find_file(name_of_module)
if coresponding_nmda_file:
ctx = create_context('{}:{}'.format(
os.path.abspath(self.__yang_models),
self.__save_file_dir))
with open(coresponding_nmda_file, 'r') as f:
a = ctx.add_module(coresponding_nmda_file, f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
try:
with open('{}/pyang_temp.txt'.format(self.temp_dir),
'w') as f:
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
with open('{}/pyang_temp.txt'.format(self.temp_dir),
'r') as f:
stdout = f.read()
except:
stdout = ""
os.unlink('{}/pyang_temp.txt'.format(self.temp_dir))
pyang_list_of_rows = stdout.split('\n')[2:]
if len(ctx.errors) != 0 and len(stdout) == 0:
return False
elif stdout == '':
return False
for x in range(0, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '':
break
if len(rows[x].split('+--')[0]) == 4:
if '-state' in rows[x]:
return False
if len(rows[x].split('augment')[0]) == 2:
part = rows[x].strip(' ').split('/')[1]
if '-state' in part:
return False
if '+--ro ' in rows[x]:
leaf = \
rows[x].split('+--ro ')[1].split(' ')[0].split(
'?')[0]
dataExist = False
for y in range(0, len(pyang_list_of_rows)):
if leaf in pyang_list_of_rows[y]:
dataExist = True
if not dataExist:
return False
return True
else:
return False
else:
return False
def __resolve_tree_type(self):
def is_openconfig(rows, output):
count_config = output.count('+-- config')
count_state = output.count('+-- state')
if count_config != count_state:
return False
row_number = 0
skip = []
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if '' == row.strip(' '):
break
if '+--rw' in row and row_number != 0 \
and row_number not in skip and '[' not in row and \
(len(row.replace('|', '').strip(' ').split(
' ')) != 2 or '(' in row):
if '->' in row and 'config' in row.split('->')[
1] and '+--rw config' not in rows[row_number - 1]:
row_number += 1
continue
if '+--rw config' not in rows[row_number - 1]:
if 'augment' in rows[row_number - 1]:
if not rows[row_number - 1].endswith(':config:'):
return False
else:
return False
length_before = set([len(row.split('+--')[0])])
skip = []
for x in range(row_number, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if len(rows[x].split('+--')[0]) not in length_before:
if (len(rows[x].replace('|', '').strip(' ').split(
' ')) != 2 and '[' not in rows[x]) \
or '+--:' in rows[x] or '(' in rows[x]:
length_before.add(len(rows[x].split('+--')[0]))
else:
break
if '+--ro' in rows[x]:
return False
duplicate = \
rows[x].replace('+--rw', '+--ro').split('+--')[1]
if duplicate.replace(' ', '') not in output.replace(
' ', ''):
return False
skip.append(x)
if '+--ro' in row and row_number != 0 and row_number not in skip and '[' not in row and \
(len(row.replace('|', '').strip(' ').split(
' ')) != 2 or '(' in row):
if '->' in row and 'state' in row.split(
'->')[1] and '+--ro state' not in rows[row_number -
1]:
row_number += 1
continue
if '+--ro state' not in rows[row_number - 1]:
if 'augment' in rows[row_number - 1]:
if not rows[row_number - 1].endswith(':state:'):
return False
else:
return False
length_before = len(row.split('+--')[0])
skip = []
for x in range(row_number, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if len(rows[x].split('+--')[0]) < length_before:
break
if '+--rw' in rows[x]:
return False
skip.append(x)
row_number += 1
return True
def is_combined(rows, output):
if output.split('\n')[1].endswith('-state'):
return False
next_obsolete_or_deprecated = False
for row in rows:
if next_obsolete_or_deprecated:
if 'x--' in row or 'o--' in row:
next_obsolete_or_deprecated = False
else:
return False
if 'x--' in row or 'o--' in row:
continue
if '+--rw config' == row.replace(
'|', '').strip(' ') or '+--ro state' == row.replace(
'|', '').strip(' '):
return False
if len(row.split('+--')[0]) == 4:
if '-state' in row and '+--ro' in row:
return False
if 'augment' in row and len(row.split('augment')[0]) == 2:
part = row.strip(' ').split('/')[1]
if '-state' in part:
next_obsolete_or_deprecated = True
part = row.strip(' ').split('/')[-1]
if ':state:' in part or '/state:' in part \
or ':config:' in part or '/config:' in part:
next_obsolete_or_deprecated = True
return True
def is_transational(rows, output):
if output.split('\n')[1].endswith('-state'):
if '+--rw' in output:
return False
if output.startswith('\n'):
name_of_module = output.split('\n')[1].split(': ')[1]
else:
name_of_module = output.split('\n')[0].split(': ')[1]
name_of_module = name_of_module.split('-state')[0]
coresponding_nmda_file = self.__find_file(name_of_module)
if coresponding_nmda_file:
ctx = create_context('{}:{}'.format(
os.path.abspath(self.__yang_models),
self.__save_file_dir))
with open(coresponding_nmda_file, 'r') as f:
a = ctx.add_module(coresponding_nmda_file, f.read())
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
try:
with open('{}/pyang_temp.txt'.format(self.temp_dir),
'w') as f:
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
with open('{}/pyang_temp.txt'.format(self.temp_dir),
'r') as f:
stdout = f.read()
except:
stdout = ""
os.unlink('{}/pyang_temp.txt'.format(self.temp_dir))
pyang_list_of_rows = stdout.split('\n')[2:]
if len(ctx.errors) != 0 and len(stdout) == 0:
return False
elif stdout == '':
return False
for x in range(0, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '':
break
if len(rows[x].split('+--')[0]) == 4:
if '-state' in rows[x]:
return False
if len(rows[x].split('augment')[0]) == 2:
part = rows[x].strip(' ').split('/')[1]
if '-state' in part:
return False
if '+--ro ' in rows[x]:
leaf = \
rows[x].split('+--ro ')[1].split(' ')[0].split(
'?')[0]
dataExist = False
for y in range(0, len(pyang_list_of_rows)):
if leaf in pyang_list_of_rows[y]:
dataExist = True
if not dataExist:
return False
return True
else:
return False
else:
return False
def is_split(rows, output):
failed = False
row_num = 0
if output.split('\n')[1].endswith('-state'):
return False
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if '+--rw config' == row.replace('|', '').strip(
' ') or '+--ro state' == row.replace('|', '') \
.strip(' '):
return False
if 'augment' in row:
part = row.strip(' ').split('/')[-1]
if ':state:' in part or '/state:' in part or ':config:' in part or '/config:' in part:
return False
for row in rows:
if 'x--' in row or 'o--' in row:
continue
if row == '':
break
if (len(row.split('+--')[0]) == 4
and 'augment' not in rows[row_num - 1]) or len(
row.split('augment')[0]) == 2:
if '-state' in row:
if 'augment' in row:
part = row.strip(' ').split('/')[1]
if '-state' not in part:
row_num += 1
continue
for x in range(row_num + 1, len(rows)):
if 'x--' in rows[x] or 'o--' in rows[x]:
continue
if rows[x].strip(' ') == '' \
or (len(rows[x].split('+--')[
0]) == 4 and 'augment' not in
rows[row_num - 1]) \
or len(row.split('augment')[0]) == 2:
break
if '+--rw' in rows[x]:
failed = True
break
row_num += 1
if failed:
return False
else:
return True
x = 0
for module in self.__all_modules['module']:
x += 1
self.__path = '{}/{}@{}.yang'.format(self.__save_file_dir,
module['name'],
module['revision'])
LOGGER.info('Searching tree type for {}. {} out of {}'.format(
module['name'], x, len(self.__all_modules['module'])))
LOGGER.debug('Get tree type from tag from module {}'.format(
self.__path))
ctx = create_context('{}:{}'.format(
os.path.abspath(self.__yang_models), self.__save_file_dir))
with open(self.__path, 'r', errors='ignore') as f:
a = ctx.add_module(self.__path, f.read())
if a is None:
LOGGER.debug(
'Could not use pyang to generate tree because of errors on module {}'
.format(self.__path))
module['tree-type'] = 'unclassified'
self.__new_modules.append(module)
continue
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
try:
with open('{}/pyang_temp.txt'.format(self.temp_dir), 'w') as f:
emit_tree(ctx, [a], f, ctx.opts.tree_depth,
ctx.opts.tree_line_length, path)
with open('{}/pyang_temp.txt'.format(self.temp_dir), 'r') as f:
stdout = f.read()
except:
module['tree-type'] = 'not-applicable'
continue
os.unlink('{}/pyang_temp.txt'.format(self.temp_dir))
if stdout == '':
module['tree-type'] = 'not-applicable'
else:
if stdout.startswith('\n'):
pyang_list_of_rows = stdout.split('\n')[2:]
else:
pyang_list_of_rows = stdout.split('\n')[1:]
if 'submodule' == module['module-type']:
LOGGER.debug('Module {} is a submodule'.format(
self.__path))
module['tree-type'] = 'not-applicable'
elif is_combined(pyang_list_of_rows, stdout):
module['tree-type'] = 'nmda-compatible'
elif is_transational(pyang_list_of_rows, stdout):
module['tree-type'] = 'transitional-extra'
elif is_openconfig(pyang_list_of_rows, stdout):
module['tree-type'] = 'openconfig'
elif is_split(pyang_list_of_rows, stdout):
module['tree-type'] = 'split'
else:
module['tree-type'] = 'unclassified'
self.__new_modules.append(module)
def create_diff_tree(name1: str, revision1: str, file2: str, revision2: str):
"""Create preformated HTML which contains diff between two yang trees.
Dump content of yang files into tempporary schema-tree-diff.txt file.
Make GET request to URL https://www.ietf.org/rfcdiff/rfcdiff.pyht?url1=<file1>&url2=<file2>'.
Output of rfcdiff tool then represents response of the method.
Arguments:
:param name1: (str) name of the first module
:param revision1: (str) revision of the first module in format YYYY-MM-DD
:param name2: (str) name of the second module
:param revision2: (str) revision of the second module in format YYYY-MM-DD
:return preformatted HTML with corresponding data
"""
schema1 = '{}/{}@{}.yang'.format(ac.d_save_file_dir, name1, revision1)
schema2 = '{}/{}@{}.yang'.format(ac.d_save_file_dir, file2, revision2)
plugin.plugins = []
plugin.init([])
ctx = create_context('{}:{}'.format(ac.d_yang_models_dir,
ac.d_save_file_dir))
ctx.opts.lint_namespace_prefixes = []
ctx.opts.lint_modulename_prefixes = []
ctx.lax_quote_checks = True
ctx.lax_xpath_checks = True
for p in plugin.plugins:
p.setup_ctx(ctx)
with open(schema1, 'r') as ff:
a = ctx.add_module(schema1, ff.read())
ctx.errors = []
if ctx.opts.tree_path is not None:
path = ctx.opts.tree_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = None
ctx.validate()
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth, ctx.opts.tree_line_length,
path)
stdout = f.getvalue()
file_name1 = 'schema1-tree-diff.txt'
full_path_file1 = '{}/{}'.format(ac.w_save_diff_dir, file_name1)
with open(full_path_file1, 'w+') as ff:
ff.write('<pre>{}</pre>'.format(stdout))
with open(schema2, 'r') as ff:
a = ctx.add_module(schema2, ff.read())
ctx.validate()
f = io.StringIO()
emit_tree(ctx, [a], f, ctx.opts.tree_depth, ctx.opts.tree_line_length,
path)
stdout = f.getvalue()
file_name2 = 'schema2-tree-diff.txt'
full_path_file2 = '{}/{}'.format(ac.w_save_diff_dir, file_name2)
with open(full_path_file2, 'w+') as ff:
ff.write('<pre>{}</pre>'.format(stdout))
tree1 = '{}/compatibility/{}'.format(ac.w_my_uri, file_name1)
tree2 = '{}/compatibility/{}'.format(ac.w_my_uri, file_name2)
diff_url = (
'https://www.ietf.org/rfcdiff/rfcdiff.pyht?url1={}&url2={}'.format(
tree1, tree2))
response = requests.get(diff_url)
os.unlink(full_path_file1)
os.unlink(full_path_file2)
return '<html><body>{}</body></html>'.format(response.text)
