def print_dict(v, module, prefix='', level=0, huff=[]):
#huff.append(Statement(None, None , None, 'container','mqtt-netconf-bridge'))
#module.substmts.append(huff[count])
global count, level_memory
if isinstance(v, dict):
for k, v2 in v.items():
p2 = "{}['{}']".format(prefix, k)
print("\nDict: --- ", level, count)
print(k, v2)
huff.append(Statement(None, None, None, 'container', k))
if count is 0:
module.substmts.append(huff[level])
level_memory[level] = count
elif count is not 0 and level is 0:
huff[level_memory[level]].substmts.append(huff[count])
#level_memory[level] = count
elif k is 'rpc':
level = 0
module.substmts.append(huff[count])
else:
print('D:', level, repr(huff[1]))
huff[level_memory[level - 1]].substmts.append(huff[count])
#huff[level-1].substmts.append(huff[count])
level_memory[level] = count
count += 1
#if count > level:
# level = count-1
print_dict(v2, module, p2, level=level + 1, huff=huff)
elif isinstance(v, list):
for i, v2 in enumerate(v):
p2 = "{}[{}]".format(prefix, i)
print("List: ---", level)
print(i, v2)
#
huff.append(Statement(None, None, None, 'list', i))
# huff[level-1].substmts.append(huff[level])
huff[level_memory[level - 1]].substmts.append(huff[count])
count += 1
print_dict(v2, module, p2, level=level + 1, huff=huff)
else:
print('{} = {}'.format(prefix, repr(v)))
huff.append(Statement(None, None, None, 'leaf', v))
#huff[level-1].substmts.append(huff[count])
huff[level_memory[level - 1]].substmts.append(huff[count])
level_memory[level] = count
count += 1
def mtype_to_statement(cls, mtype, yangname, parent=None):
if ismodeledclass(mtype):
return YANGContainer[mtype].to_statement(
yangname=yangname, parent=parent)
leaf = Statement(None, parent, None, 'leaf', yangname)
# if member.title:
# description = Statement(
# None, leaf, None, 'description', member.title)
# leaf.substmts.append(description)
type_ = Statement(None, leaf, None, 'type', TYPES[mtype])
leaf.substmts.append(type_)
return leaf
def test_dump():
"""
dump should correctly print headless pyang.statements.Statement
dump should correctly print nested pyang.statements.Statement
"""
prefix = Statement(None, None, None, 'prefix', 'test')
assert dump(prefix).strip() == 'prefix test;'
namespace = Statement(None, None, None, 'namespace', 'urn:yang:test')
module = Statement(None, None, None, 'module', 'test')
module.substmts = [namespace, prefix]
assert dump(module).strip() == (
'module test {\n'
' namespace "urn:yang:test";\n'
' prefix test;\n'
'}'
)
def fixup_state_grouping(stmt):
# Add config: false to every statement that doesn't already have it.
# Recurse over groupings statements.
if stmt.keyword in {"grouping"}:
for s in stmt.substmts:
fixup_state_grouping(s)
# Fixup any data nodes that can have a status leaf.
if stmt.keyword in {
"container", "leaf", "leaf-list", "list", "anyxml", "anydata",
"choice"
}:
has_config_true = False
has_config_false = False
for s in stmt.substmts:
if s.keyword == 'config':
if s.arg == 'true':
has_config_true = True
else:
has_config_false = True
if not has_config_false:
add_substmt_canonical(
stmt, Statement(stmt.top, stmt, stmt.pos, 'config', 'false'))
if has_config_true:
pass
for s in stmt.substmts:
fixup_state_grouping(s)
def convert_stmt(config_stmt, state_stmt):
# Iterate the state tree
state_substmts = grammar.sort_canonical(state_stmt.keyword,
state_stmt.substmts)
config_substmts = grammar.sort_canonical(config_stmt.keyword,
config_stmt.substmts)
for s in state_substmts:
matching_cfg_stmts = [
c for c in config_substmts if is_matching_stmt(c, s)
]
if any(matching_cfg_stmts):
cfg_stmt = matching_cfg_stmts[0]
if (s.keyword in ['description', 'presence']):
if s.arg != cfg_stmt.arg:
# Merge description strings.
cfg_stmt.arg = cfg_stmt.arg + "\n\nFROM STATE TREE (FIX ME):\n" + s.arg
else:
# Don't add this element (should also check type as well).
convert_stmt(cfg_stmt, s)
else:
if (s.keyword != 'config'):
# Copy the state stmt over to the config
copied_stmt = s.copy()
config_stmt.substmts.append(copied_stmt)
fixup_stmt(copied_stmt)
if copied_stmt.keyword in {
"container", "leaf", "leaf-list", "list", "anyxml",
"anydata", "choice"
}:
add_substmt_canonical(
copied_stmt,
Statement(copied_stmt.top, copied_stmt,
copied_stmt.pos, 'config', 'false'))
def check_namespace(parsed_module: Statement) -> bool:
try:
namespace = parsed_module.search('namespace')[0].arg
except:
return False
if 'urn:cisco' in namespace:
return False
if 'urn:' in namespace:
return True
for ns, _ in NS_MAP:
if ns in namespace:
return True
return False
def container():
"""Sample container"""
container = Statement(None, None, None, 'container', 'outer')
id_ = Statement(None, container, None, 'leaf', 'id')
type_ = Statement(None, id_, None, 'type', 'int32')
id_.substmts = [type_]
name = Statement(None, container, None, 'leaf', 'name')
type_ = Statement(None, name, None, 'type', 'string')
name.substmts = [type_]
container.substmts = [id_, name]
return container
def module():
"""Sample container"""
module = Statement(None, None, None, 'module', 'fixture-test')
namespace = Statement(module, module, None, 'namespace', 'urn:yang:test')
prefix = Statement(module, module, None, 'prefix', 'test')
container = Statement(module, module, None, 'container', 'outer')
id_ = Statement(module, container, None, 'leaf', 'id')
type_ = Statement(module, id_, None, 'type', 'int32')
id_.substmts = [type_]
name = Statement(module, container, None, 'leaf', 'name')
type_ = Statement(module, name, None, 'type', 'string')
name.substmts = [type_]
container.substmts = [id_, name]
module.substmts = [namespace, prefix, container]
return module
def check_dependencies(dep_type: t.Literal['import', 'include'], parsed_module: Statement,
directory: str, yang_models_dir: str) -> t.Tuple[t.Set[str], t.Set[str]]:
all_modules: t.Set[str] = set()
missing_modules: t.Set[str] = set()
for dependency in parsed_module.search(dep_type):
name = dependency.arg
all_modules.add(name)
revisions = dependency.search('revision-date')
revision = revisions[0].arg if revisions else '*'
pattern = '{}.yang'.format(name)
pattern_with_revision = '{}@{}.yang'.format(name, revision)
if not find_first_file(directory, pattern, pattern_with_revision, yang_models_dir):
# TODO: if the matched filename doesn't include the revision, maybe we should check it?
if revision != '*':
missing_modules.add('{}@{}'.format(name, revision))
else:
missing_modules.add(name)
return all_modules, missing_modules
def check_revision(parsed_module: Statement) -> bool:
try:
revision = parsed_module.search('revision')[0].arg
except:
return False
revision_parts = [int(i) for i in revision.split('-')]
try:
date(*revision_parts)
except:
if revision_parts[1:] == [2, 29]:
revision_parts[2] = 28
try:
date(*revision_parts)
except:
return False
else:
return False
return True
def test_mix_builder(Y):
"""
builder should mix with pyang standard
"""
module = Y('module', 'test',
Statement(None, None, None, 'namespace', 'urn:yang:test'))
module('prefix', 'test')
module.append(Y.from_tuple(('leaf', 'data', [('type', 'string')])))
assert module.dump().strip() == ('module test {\n'
' namespace "urn:yang:test";\n'
' prefix test;\n'
' leaf data {\n'
' type string;\n'
' }\n'
'}')
assert module.validate()
def to_statement(cls, yangname=None, parent=None):
"""Get YANG container as :class:`pyang.statement.Statement` instance,
ready for feeding to a pyang output plugin.
- If no `yangname` is given,
a decamelized modeled class name joined by hyphens will be used.
- Optionally binds statement to `parent` statement,
like a module or a parent container.
"""
container = Statement(
None, parent, None, 'container', yangname or cls.yangname)
# add modeled members as leafs or subcontainers to the main container
for name, member in cls.mclass.model.members:
yangname = name.replace('_', '-')
statement = cls.member_to_statement(
member, yangname=yangname, parent=container)
container.substmts.append(statement)
return container
def to_statement(cls, namespace=None, prefix=None, revision=None):
"""Get YANG module as :class:`pyang.statement.Statement` instance,
ready for feeding to a pyang output plugin.
"""
# adapted modeled class is used as YANG module and main container
module = Statement(None, None, None, 'module', cls.yangname)
if not prefix:
prefix = decamelize(cls.__name__, joiner='-')
if namespace:
namespace = Statement(None, module, None, 'namespace', namespace)
module.substmts.append(namespace)
prefix = Statement(None, module, None, 'prefix', prefix)
module.substmts.append(prefix)
if not revision:
revision = str(date.today())
revision = Statement(None, module, None, 'revision', revision)
module.substmts.append(revision)
container = super(YANGModuleMeta, cls).to_statement(parent=module)
module.substmts.append(container)
# add defined @rpc methods
for name, obj in getmembers(cls):
if getattr(obj, '__isnetconfrpc__', False):
method = obj
yangname = name.replace('_', '-')
rpc = Statement(None, module, None, 'rpc', yangname)
module.substmts.append(rpc)
if method.__doc__:
description = Statement(None, rpc, None, 'description',
method.__doc__.strip())
rpc.substmts.append(description)
# if the rpc method has args then add an input statement,
# containing a leaf statement for every arg
argspec = getargspec(method)
if len(argspec.args) > 1: # ignore 'self'
input_ = Statement(None, rpc, None, 'input', None)
rpc.substmts.append(input_)
for name in argspec.args[1:]:
yangname = name.replace('_', '-')
mtype = method.mtypes[name]
statement = cls.member_to_statement(mtype,
yangname=yangname,
parent=input_)
input_.substmts.append(statement)
return module
def member_to_statement(cls, member, yangname, parent=None):
if ismodeledclass(member.mtype):
return YANGContainer[member.mtype].to_statement(
yangname=yangname, parent=parent)
if member.islist():
list_ = Statement(None, parent, None, 'list', yangname)
yangindexname = member.indexname.replace('_', '-')
key = Statement(None, list_, None, 'key', yangindexname)
list_.substmts.append(key)
leaf = Statement(None, list_, None, 'leaf', yangindexname)
list_.substmts.append(leaf)
type_ = Statement(None, leaf, None, 'type', TYPES[int])
leaf.substmts.append(type_)
yangitemname = member.itemname.replace('_', '-')
statement = cls.mtype_to_statement(
member.itemtype, yangname=yangitemname, parent=list_)
list_.substmts.append(statement)
return list_
if member.isdict():
list_ = Statement(None, parent, None, 'list', yangname)
yangkeyname = member.keyname.replace('_', '-')
key = Statement(None, list_, None, 'key', yangkeyname)
list_.substmts.append(key)
leaf = Statement(None, list_, None, 'leaf', yangkeyname)
list_.substmts.append(leaf)
type_ = Statement(
None, leaf, None, 'type', TYPES[member.keytype])
leaf.substmts.append(type_)
yangvaluename = member.valuename.replace('_', '-')
statement = cls.mtype_to_statement(
member.valuetype, yangname=yangvaluename, parent=list_)
list_.substmts.append(statement)
return list_
return cls.mtype_to_statement(
member.mtype, yangname=yangname, parent=parent)
def mark_stmt_deprecated(state_stmt):
# Iterate the state tree
def can_have_status(stmt):
return (stmt.keyword in [
'action', 'anydata', 'anyxml', 'augment', 'bits', 'case', 'choice',
'container', 'enum', 'extension', 'feature', 'grouping',
'identity', 'leaf', 'list', 'leaf-list', 'notification', 'rpc',
'typedef', 'uses'
])
if (can_have_status(state_stmt)):
status_current = True
for substmt in state_stmt.substmts:
if (substmt.keyword == 'status' and substmt.arg != "current"):
status_current = False
if (status_current):
add_substmt_canonical(
state_stmt,
Statement(state_stmt.top, state_stmt, state_stmt.pos, 'status',
'deprecated'))
# Recurse down children state statements.
for substmt in state_stmt.substmts:
mark_stmt_deprecated(substmt)
def test_append(Y, container):
"""
Append should accept N (mixed) arguments
Append without copy should reuse node
Append with copy should build a new node
"""
other_ext = Statement(None, None, None, 'other:ext', 'other:ipsun')
container.append(Y.description('loren ipsum'), Y('ext:loren', 'ipsun'),
other_ext)
assert container.find('description')
assert container.find('ext:loren')
assert len(container.find(arg='ipsun', ignore_prefix=True)) == 2
result = container.find('other:ext')[0].unwrap()
assert id(result) == id(other_ext)
container.append(Y.description('loren ipsum'),
Y('ext:loren', 'ipsun'),
other_ext,
copy=True)
result = container.find('other:ext')[-1].unwrap()
assert id(result) != id(other_ext)
def parse_dict(v, module):
global mqtt_commands, my_set, device_category
print("##############")
huff2 = []
count2 = 0
if isinstance(v, dict):
for k, v2 in v.items():
print("# K:", k)
if k == 'device':
huff2.append(Statement(None, None, None, 'container', k))
module.substmts.append(huff2[count2])
count2 += 1
level = count2 - 1
for k2, v3 in v2.items():
print("#### K2:", k2, count2)
if k2 == 'description':
huff2.append(Statement(None, None, None, k2, v3))
huff2[level].substmts.append(huff2[count2])
count2 += 1
elif k2 == 'uuid':
huff2.append(
Statement(None, None, None, 'list', 'device-id'))
huff2[level].substmts.append(huff2[count2])
count2 += 1
level2 = count2 - 1
huff2.append(Statement(None, None, None, 'key', k2))
huff2[level2].substmts.append(huff2[count2])
count2 += 1
huff2.append(Statement(None, None, None, 'leaf', k2))
huff2[level2].substmts.append(huff2[count2])
count2 += 1
for k3, v4 in v3.items():
print("+++++++ K3:", k3, v4, count2)
if k3 == 'value':
my_set.add(
v4) #add this value to the set of UUIDs
print(my_set)
else:
huff2.append(
Statement(None, None, None, k3, v4))
huff2[count2 - 1].substmts.append(
huff2[count2])
count2 += 1
else:
huff2.append(Statement(None, None, None, 'leaf', k2))
huff2[level].substmts.append(huff2[count2])
count2 += 1
level2 = count2 - 1
for k3, v4 in v3.items():
print("###### K3:", k3, v4, count2)
if k3 == 'value': #special rule for VALUE in 'device-category'
device_category = v4 #add this value to the set of UUIDs
print(device_category)
elif not isinstance(v4, dict):
huff2.append(
Statement(None, None, None, k3, v4))
huff2[level2].substmts.append(huff2[count2])
count2 += 1
else:
#key, value = v4.popitem()
#print '>>>',key, value
huff2.append(
Statement(None, None, None, k3, None))
huff2[count2 - 1].substmts.append(
huff2[count2])
count2 += 1
for k4, v5 in v4.items():
print("########x K4:", k4, v5, count2)
huff2.append(
Statement(None, None, None, k4, v5))
huff2[count2 - 1].substmts.append(
huff2[count2])
count2 += 1
if k == 'rpc':
for k2, v3 in v2.items():
print("#### K2:", k2, count2)
huff2.append(Statement(None, None, None, 'rpc', k2))
module.substmts.append(huff2[count2])
count2 += 1
level = count2 - 1
for k3, v4 in v3.items():
print("###### K3:", k3, count2)
if not isinstance(v4, dict):
if k3 == 'mqtt-command':
mqtt_commands[
k2] = v4 # add rpc-name : mqtt-command
else:
print("K3 - append", k3, v4)
huff2.append(
Statement(None, None, None, k3, v4))
huff2[level].substmts.append(huff2[count2])
count2 += 1
else:
if k3 == 'input':
huff2.append(
Statement(None, None, None, 'input', None))
huff2[level].substmts.append(huff2[count2])
count2 += 1
huff2.append(
Statement(None, None, None, 'leaf',
next(iter(v4))))
huff2[count2 - 1].substmts.append(
huff2[count2])
level2 = count2
count2 += 1
for k4, v5 in v4.items():
print("******** K4:", k4, v5, count2)
if not isinstance(v4, dict):
print(
"!!! Fehler: sollte nicht vorkommen"
)
huff2.append(
Statement(None, None, None, 'ooh',
k4))
huff2[count2 - 1].substmts.append(
huff2[count2])
count2 += 1
else:
for k5, v6 in v5.items():
print("######## K5:", k5, v6,
count2)
huff2.append(
Statement(
None, None, None, k5, v6))
huff2[level2].substmts.append(
huff2[count2])
count2 += 1
def generate_yang(test, uuid_set):
global count, level_memory
global mqtt_commands
global my_set
global device_category
my_set = uuid_set
'''
Generate a YANG-in-XML Tree
- print the YANG Tree as string with SerialIO
- build a new root-Element, called <data> with 'xmlns' Attribute
- attach the stringified CDATA to the new Element
- print the XML
'''
#python-modeled.netconf/modeled/netconf/yang/__init__.py
module1 = Statement(None, None, None, 'module', 'mqtt-led')
my_namespace = "http://ipv6lab.beuth-hochschule.de/led"
my_prefix = "led"
namespace = Statement(None, module1, None, 'namespace', my_namespace)
module1.substmts.append(namespace)
prefix = Statement(None, module1, None, 'prefix', my_prefix)
module1.substmts.append(prefix)
#http://stackoverflow.com/questions/10844064/items-in-json-object-are-out-of-order-using-json-dumps
data = json.loads(test, object_pairs_hook=OrderedDict)
count = 0
level_memory = {}
#print_dict(data, module1, count)
parse_dict(data, module1)
#revision = str(datetime.now())
#revision = Statement(None, module, None, 'revision', revision)
#module.substmts.append(revision)
#https://github.com/mbj4668/pyang/blob/master/pyang/plugin.py
#https://github.com/modeled/modeled.netconf/blob/master/modeled/netconf/yang/container.py
"""Serialize YANG container to the given output `format`.
"""
# output stream for pyang output plugin
stream = StringIO()
# gets filled with all availabe pyang output format plugins
PYANG_PLUGINS = {}
# register and initialise pyang plugin
pyang.plugin.init([])
for plugin in pyang.plugin.plugins:
plugin.add_output_format(PYANG_PLUGINS)
del plugin
#for name in PYANG_PLUGINS:
# print(name)
#...
#dsdl
#depend
#name
#omni
#yin
#tree
#jstree
#capability
#yang
#uml
#jtox
#jsonxsl
#sample-xml-skeleton
plugin = PYANG_PLUGINS['yang']
# register plugin options according to pyang script
optparser = OptionParser()
plugin.add_opts(optparser)
# pyang plugins also need a pyang.Context
ctx = pyang.Context(DummyRepository())
# which offers plugin-specific options (just take defaults)
ctx.opts = optparser.parse_args([])[0]
# ready to serialize
plugin.emit(ctx, [module1], stream)
# and return the resulting data
stream.seek(0)
yang = stream.getvalue()
print('\nAusgabe: ')
print(stream.read())
print("")
#return stream.read()
#root = etree.Element("data", xmlns="urn:ietf:params:xml:ns:yang:ietf-netconf-monitoring")
#root.text = etree.CDATA(yang)
#print etree.tostring(root, pretty_print=True)
#return etree.tostring(root, pretty_print=True)
#returns the constructed yang from json-config, a list of mqtt-commands and a set of uuids
return (yang, mqtt_commands, my_set, device_category)
def convert_stmt(ctx, stmt, level):
if ctx.opts.yang_remove_unused_imports and stmt.keyword == 'import':
for p in stmt.parent.i_unused_prefixes:
if stmt.parent.i_unused_prefixes[p] == stmt:
return
if util.is_prefixed(stmt.raw_keyword):
(prefix, identifier) = stmt.raw_keyword
keyword = prefix + ':' + identifier
else:
keyword = stmt.keyword
if keyword == 'module':
# Change the module name.
module_name = stmt.arg
stmt.arg = module_name + '-state'
# Find the module prefix.
prefix_stmt = next(x for x in stmt.substmts if x.keyword == 'prefix')
# Rename the prefix statement.
prefix = prefix_stmt.arg
prefix_stmt.arg = prefix + '-s'
# Add an import statement back to the original module.
import_stmt = Statement(stmt.top, stmt, stmt.pos, 'import',
module_name)
add_substmt_canonical(stmt, import_stmt)
add_substmt_canonical(
import_stmt,
Statement(stmt.top, import_stmt, import_stmt.pos, 'prefix',
prefix))
if keyword == 'submodule':
# Change the module name.
submodule_name = stmt.arg
stmt.arg = submodule_name + '-state'
if keyword == 'namespace':
stmt.arg = stmt.arg + '-state'
# Remove any feature statements, reference the original module feature instead.
if keyword == 'feature':
stmt.parent.substmts.remove(stmt)
if keyword == 'if-feature':
fix_references(stmt, stmt.i_module.i_features)
# Remove any identity statements, reference the original identity instead.
# Identity base won't matter because they are all removed.
if keyword == 'identity':
stmt.parent.substmts.remove(stmt)
if keyword in ('include', 'belongs-to'):
stmt.arg = stmt.arg + "-state"
if keyword == "prefix" and stmt.parent.keyword == "belongs-to":
stmt.arg = stmt.arg + "-s"
# Remove must/when statements.
if keyword in ('must', 'when'):
stmt.parent.substmts.remove(stmt)
#fix_references(stmt, stmt.i_module.i_identities)
if keyword == 'type' and stmt.arg == 'identityref':
base_stmt = next(x for x in stmt.substmts if x.keyword == 'base')
fix_references(base_stmt, stmt.i_module.i_identities)
# Remove any typedef statements, reference the original typedef instead.
if keyword == 'typedef':
stmt.parent.substmts.remove(stmt)
if keyword == 'type':
fix_references(stmt, stmt.i_module.i_typedefs)
# Remove all config statements, only the top level config false is necessary.
if keyword == 'config':
stmt.parent.substmts.remove(stmt)
if len(stmt.substmts) != 0:
#substmts = grammar.sort_canonical(stmt.keyword, stmt.substmts)
for s in stmt.substmts[:]:
convert_stmt(ctx, s, level + 1)
# Convert top level containers from "foo" to "foo-state", and mark it as config false.
if keyword in ('container', 'list',
'leaf-list') and (stmt.parent.keyword == 'module'
or stmt.parent.keyword == 'grouping'):
if not stmt.arg.endswith('-state'):
#stmt.arg = stmt.arg + '-state'
add_substmt_canonical(
stmt, Statement(stmt.top, stmt, stmt.pos, 'config', 'false'))