def emit_json_xsl(modules, ctx, fd):
"""Main control function.
Set up the top-level parts of the stylesheet, the process
recursively all nodes in all data trees, and finally emit the
serialized stylesheet.
"""
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("JSONXSL plugin needs a valid module")
tree = ET.ElementTree(ss)
ET.SubElement(ss, "output", method="text")
xsltdir = os.environ.get("PYANG_XSLT_DIR", "/usr/local/share/yang/xslt")
ET.SubElement(ss, "include", href=xsltdir + "/jsonxsl-templates.xsl")
ET.SubElement(ss, "strip-space", elements="*")
nsmap = ET.SubElement(ss, "template", name="nsuri-to-module")
ET.SubElement(nsmap, "param", name="uri")
choo = ET.SubElement(nsmap, "choose")
for module in modules:
ns_uri = module.search_one("namespace").arg
ss.attrib["xmlns:" + module.i_prefix] = ns_uri
when = ET.SubElement(choo, "when", test="$uri='" + ns_uri + "'")
xsl_text(module.i_modulename, when)
process_module(module)
tree.write(fd, encoding="utf-8", xml_declaration=True)
def _validate_pyang_modules(self, filenames):
# all the module have been added so get the context to validate
# call prevalidate before this and post validate after
self.ctx.validate()
def keyfun(e):
if e[0].ref == filenames[0]:
return 0
else:
return 1
self.ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
if len(filenames) > 0:
# first print error for the first filename given
self.ctx.errors.sort(key=keyfun)
error_messages = []
for (epos, etag, eargs) in self.ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
logger.warning('%s: %s\n' %
(str(epos), error.err_to_str(etag, eargs)))
else:
err_msg = '%s: %s\n' % (str(epos), error.err_to_str(
etag, eargs))
logger.error(err_msg)
error_messages.append(err_msg)
if len(error_messages) > 0:
err_msg = '\n'.join(error_messages)
raise YdkGenException(err_msg)
def _validate_pyang_modules(self, filenames):
# all the module have been added so get the context to validate
# call prevalidate before this and post validate after
self.ctx.validate()
def keyfun(e):
if e[0].ref == filenames[0]:
return 0
else:
return 1
self.ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
if len(filenames) > 0:
# first print error for the first filename given
self.ctx.errors.sort(key=keyfun)
error_messages = []
for (epos, etag, eargs) in self.ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
logger.warning('%s: %s\n' %
(str(epos), error.err_to_str(etag, eargs)))
else:
err_msg = '%s: %s\n' % (str(epos), error.err_to_str(etag, eargs))
logger.error(err_msg)
error_messages.append(err_msg)
if len(error_messages) > 0:
err_msg = '\n'.join(error_messages)
raise YdkGenException('''\nError occured: "%s". \nThe models supplied to the YDK generator are invalid. Please make sure the models are valid by compiling the models together using pyang. Please run "pyang *" in the models directory, make sure there are no errors and then try running the generator again. If there are model errors, please fix the errors by editing the model, contacting the model owner or deleting the model from the list of models to generate the YDK bindings for.'''%err_msg)
def emit(self, ctx, modules, fd):
modulenames = [m.arg for m in modules]
if not ctx.opts.ignore_errors:
for (epos, etag, eargs) in ctx.errors:
if (epos.top.arg in modulenames
and error.is_error(error.err_level(etag))):
raise error.EmitError("%s contains errors" % epos.top.arg)
emit_docs(ctx, modules, fd)
def emit(self, ctx, modules, fd):
# cannot do this unless everything is ok for our module
modulenames = [m.arg for m in modules]
for (epos, etag, eargs) in ctx.errors:
if (epos.top.arg in modulenames
and error.is_error(error.err_level(etag))):
raise error.EmitError("%s contains errors" % epos.top.arg)
emit_depend(ctx, modules, fd)
def emit(self, ctx, modules, fd):
"""Main control function.
Set up the top-level parts of the sample document, then process
recursively all nodes in all data trees, and finally emit the
sample XML document.
"""
if ctx.opts.sample_path is not None:
path = ctx.opts.sample_path.split('/')
if path[0] == '':
path = path[1:]
else:
path = []
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError(
"sample-xml-skeleton plugin needs a valid module")
self.doctype = ctx.opts.doctype
if self.doctype not in ("config", "data"):
raise error.EmitError("Unsupported document type: %s" %
self.doctype)
self.annots = ctx.opts.sample_annots
self.defaults = ctx.opts.sample_defaults
self.node_handler = {
"container": self.container,
"leaf": self.leaf,
"anyxml": self.anyxml,
"choice": self.process_children,
"case": self.process_children,
"list": self.list,
"leaf-list": self.leaf_list,
"rpc": self.ignore,
"action": self.ignore,
"notification": self.ignore
}
self.ns_uri = {}
for yam in modules:
self.ns_uri[yam] = yam.search_one("namespace").arg
self.top = etree.Element(
self.doctype, {"xmlns": "urn:ietf:params:xml:ns:netconf:base:1.0"})
tree = etree.ElementTree(self.top)
for yam in modules:
self.process_children(yam, self.top, None, path)
if sys.version > "3":
fd.write(
str(
etree.tostring(tree,
pretty_print=True,
encoding="UTF-8",
xml_declaration=True), "UTF-8"))
elif sys.version > "2.7":
tree.write(fd,
encoding="UTF-8",
pretty_print=True,
xml_declaration=True)
else:
tree.write(fd, pretty_print=True, encoding="UTF-8")
def emit(self, ctx, modules, fd):
# cannot do this unless everything is ok for our module
modulenames = [m.arg for m in modules]
for (epos, etag, eargs) in ctx.errors:
if ((epos.top is None or epos.top.arg in modulenames)
and error.is_error(error.err_level(etag))):
raise error.EmitError("%s contains more fundamental errors "\
"than the pattern statements" % epos.top.arg)
emit_clean_pattern(ctx, modules, fd)
def emit_dsdl(ctx, modules, fd):
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("DSDL translation needs a valid module")
schema = HybridDSDLSchema().from_modules(modules,
ctx.opts.dsdl_no_dublin_core,
ctx.opts.dsdl_no_documentation,
ctx.opts.dsdl_record_defs, debug=0)
fd.write(schema.serialize())
def emit(self, ctx, modules, fd):
# Require error-free modules
modulenames = [m.arg for m in modules]
for (epos, etag, eargs) in ctx.errors:
if (epos.top.arg in modulenames
and error.is_error(error.err_level(etag))):
raise error.EmitError("%s contains errors" % epos.top.arg)
emit_dot(ctx, modules, fd)
def __print_pyang_output(ctx):
err = ''
for (epos, etag, eargs) in ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
kind = "warning"
else:
kind = "error"
err += str(epos) + ': %s: ' % kind + \
error.err_to_str(etag, eargs) + '\n'
return err
def __print_pyang_output(self):
err = ''
out = ''
for (epos, etag, eargs) in self.__ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
kind = 'warning'
else:
kind = 'error'
err += '{}: {}: {}\n'.format(epos, kind, error.err_to_str(etag, eargs))
return err, out
def emit_cts(ctx, module, fd):
# No errors are allowed
for (epos, etag, eargs) in ctx.errors:
if (epos.top_name == module.arg
and error.is_error(error.err_level(etag))):
raise error.EmitError("CTS translation needs a valid module")
schema = ConceptualTreeSchema().from_modules((module, ),
ctx.opts.cts_no_dublin_core,
ctx.opts.cts_no_documentation,
ctx.opts.cts_record_defs,
debug=0)
fd.write(schema.serialize())
def check_update(ctx, newmod):
oldpath = os.pathsep.join(ctx.opts.old_path)
olddir = os.path.dirname(ctx.opts.check_update_from)
if olddir == '':
olddir = '.'
oldpath += os.pathsep + olddir
oldrepo = repository.FileRepository(oldpath, use_env=False)
oldctx = context.Context(oldrepo)
oldctx.opts = ctx.opts
oldctx.lax_xpath_checks = ctx.lax_xpath_checks
oldctx.lax_quote_checks = ctx.lax_quote_checks
if ctx.opts.verbose:
print("Loading old modules from:")
for d in oldrepo.dirs:
print(" %s" % d)
print("")
for p in plugin.plugins:
p.setup_ctx(oldctx)
for oldfilename in [ctx.opts.check_update_from] + ctx.opts.old_deviation:
try:
fd = io.open(oldfilename, "r", encoding="utf-8")
text = fd.read()
except IOError as ex:
sys.stderr.write("error %s: %s\n" % (oldfilename, ex))
sys.exit(1)
if oldfilename in ctx.opts.old_deviation:
oldctx.add_module(oldfilename, text)
else:
oldmod = oldctx.add_module(oldfilename, text)
oldctx.validate()
ctx.errors.extend(oldctx.errors)
if oldmod is None:
return
for epos, etag, eargs in ctx.errors:
if (epos.ref in (newmod.pos.ref, oldmod.pos.ref)
and error.is_error(error.err_level(etag))):
return
if ctx.opts.verbose:
print("Loaded old modules:")
for x in oldrepo.get_modules_and_revisions(oldctx):
(m, r, (fmt, filename)) = x
print(" %s" % filename)
print("")
chk_module(ctx, oldmod, newmod)
def emit(self, ctx, modules, fd):
for (epos, etag, eargs) in ctx.errors:
if ((epos.top is None or epos.top.arg in self.mods)
and error.is_error(error.err_level(etag))):
self.fatal("%s contains errors" % epos.top.arg)
if ctx.opts.uml_pages_layout is not None:
if re.match('[0-9]x[0-9]', ctx.opts.uml_pages_layout) is None:
self.fatal(
"Illegal page split option %s, should be [0-9]x[0-9], example 2x2"
% ctx.opts.uml_pages_layout)
umldoc = uml_emitter(ctx)
umldoc.emit(modules, fd)
def emit(self, ctx, module, fd):
# cannot do XSD unless everything is ok for our module
for (epos, etag, eargs) in ctx.errors:
if (epos.top_name == module.arg
and error.is_error(error.err_level(etag))):
raise error.EmitError("XSD translation needs a valid module")
# we also need to have all other modules found
for pre in module.i_prefixes:
(modname, revision) = module.i_prefixes[pre]
mod = statements.modulename_to_module(module, modname, revision)
if mod == None:
raise error.EmitError("cannot find module %s, needed by XSD"
" translation" % modname)
emit_xsd(ctx, module, fd)
def emit(self, ctx, modules, fd):
modulenames = [m.arg for m in modules]
if not ctx.opts.ignore_errors:
for (epos, etag, eargs) in ctx.errors:
#
# When a module has not yet been parsed then the top.arg does
# not exist. This can be the case when an error is created early
# in the parsing process.
#
if not hasattr(epos.top, "arg"):
raise error.EmitError("%s contains errors, and was not parsed"
% (epos.ref))
if (epos.top.arg in modulenames and
error.is_error(error.err_level(etag))):
raise error.EmitError("%s contains errors" % epos.top.arg)
emit_paths(ctx, modules, fd)
def emit(self, ctx, modules, fd):
"""Main control function.
"""
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("sample-json-skeleton plugin needs a valid module")
tree = {}
self.defaults = ctx.opts.sample_defaults
self.doctype = ctx.opts.doctype
if self.doctype not in ("config", "data"):
raise error.EmitError("Unsupported document type: %s" %
self.doctype)
for module in modules:
self.process_children(module, tree, None)
json.dump(tree, fd, indent=4)
def emit(self, ctx, modules, fd):
"""Main control function.
"""
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("JTOX plugin needs a valid module")
tree = {}
mods = {}
annots = {}
for m, p in unique_prefixes(ctx).items():
mods[m.i_modulename] = [p, m.search_one("namespace").arg]
for module in modules:
for ann in module.search(("ietf-yang-metadata", "annotation")):
typ = ann.search_one("type")
annots[module.arg + ":" + ann.arg] = ("string" if typ is None
else self.base_type(typ))
for module in modules:
self.process_children(module, tree, None)
json.dump({"modules": mods, "tree": tree, "annotations": annots}, fd)
def emit(self, ctx, modules, fd):
"""Main control function.
Set up the top-level parts of the stylesheet, then process
recursively all nodes in all data trees, and finally emit the
serialized stylesheet.
"""
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("JSONXSL plugin needs a valid module")
self.real_prefix = unique_prefixes(ctx)
self.top_names = []
for m in modules:
self.top_names.extend([
c.arg for c in m.i_children
if c.keyword not in ("rpc", "notification")
])
tree = ET.ElementTree(ss)
ET.SubElement(ss, "output", method="text")
xsltdir = os.environ.get("PYANG_XSLT_DIR",
"/usr/local/share/yang/xslt")
ET.SubElement(ss, "include", href=xsltdir + "/jsonxsl-templates.xsl")
ET.SubElement(ss, "strip-space", elements="*")
nsmap = ET.SubElement(ss, "template", name="nsuri-to-module")
ET.SubElement(nsmap, "param", name="uri")
choo = ET.SubElement(nsmap, "choose")
for module in self.real_prefix.keys():
ns_uri = module.search_one("namespace").arg
ss.attrib["xmlns:" + self.real_prefix[module]] = ns_uri
when = ET.SubElement(choo, "when", test="$uri='" + ns_uri + "'")
self.xsl_text(module.i_modulename, when)
self.process_module(module)
if sys.version > "3":
tree.write(fd, encoding="unicode", xml_declaration=True)
elif sys.version > "2.7":
tree.write(fd, encoding="UTF-8", xml_declaration=True)
else:
tree.write(fd, encoding="UTF-8")
def emit_jtox(modules, ctx, fd):
"""Main control function.
"""
for (epos, etag, eargs) in ctx.errors:
if error.is_error(error.err_level(etag)):
raise error.EmitError("JTOX plugin needs a valid module")
tree = {}
prefixes = []
def unique_prefix(p):
"""Disambiguate the module prefix."""
suffix = 0
while p == "nc" or p in prefixes:
p += "%d" % suffix
suffix += 1
return p
for module in modules:
uri = module.search_one("namespace").arg
prf = unique_prefix(module.i_prefix)
prefixes.append(prf)
mods[module.i_modulename] = [prf, uri]
for module in modules:
process_children(module, tree)
json.dump({"modules": mods, "tree": tree}, fd)
def emit(self, ctx, modules, fd):
"""Generates Java classes from the YANG module supplied to pyang.
The generated classes are placed in the directory specified by the '-d'
or '--jnc-output' flag, or in "gen" if no such flag was provided,
using the 'directory' attribute of ctx. If there are existing files
in the output directory with the same name as the generated classes,
they are silently overwritten.
ctx -- Context used to get output directory, verbosity mode, error
handling policy (the ignore attribute) and whether or not a
schema file should be generated.
modules -- A list containing a module statement parsed from the YANG
module supplied to pyang.
fd -- File descriptor (ignored).
"""
if ctx.opts.debug or ctx.opts.verbose:
print('JNC plugin starting')
if not ctx.opts.ignore:
for (epos, etag, _) in ctx.errors:
if (error.is_error(error.err_level(etag)) and etag
in ('MODULE_NOT_FOUND', 'MODULE_NOT_FOUND_REV')):
self.fatal("%s contains errors" % epos.top.arg)
if (etag in ('TYPE_NOT_FOUND', 'FEATURE_NOT_FOUND',
'IDENTITY_NOT_FOUND', 'GROUPING_NOT_FOUND')):
util.print_warning(
msg=(etag.lower() + ', generated class ' +
'hierarchy might be incomplete.'),
key=etag)
else:
util.print_warning(msg=(etag.lower() + ', aborting.'),
key=etag)
self.fatal("%s contains errors" % epos.top.arg)
# Sweep, adding included and imported modules, until no change
module_set = set(modules)
num_modules = 0
while num_modules != len(module_set):
num_modules = len(module_set)
for module in list(module_set):
imported = map(lambda x: x.arg, util.search(module, 'import'))
included = map(lambda x: x.arg, util.search(module, 'include'))
for (module_stmt, rev) in self.ctx.modules:
if module_stmt in chain(imported, included):
module_set.add(self.ctx.modules[(module_stmt, rev)])
# Generate files from main modules
for module in filter(lambda s: s.keyword == 'module', module_set):
self.generate_from(module)
# Generate files from augmented modules
for aug_module in context.augmented_modules.values():
self.generate_from(aug_module)
# Print debug messages saying that we're done.
if ctx.opts.debug or ctx.opts.verbose:
if not self.ctx.opts.no_classes:
print('Java classes generation COMPLETE.')
if not self.ctx.opts.no_schema:
print('Schema generation COMPLETE.')
def _parse_and_return_modules(resolved_model_dir):
""" Use pyang to parse the files and get a list of modules.
:param str resolved_model_dir The directory where all models to be compiled are found.
:raise YdkGenException If there was a problem parsing the modules
"""
repos = pyang.FileRepository(resolved_model_dir, False)
ctx = pyang.Context(repos)
statements.add_validation_fun('reference_3', ['deviation'],
_add_i_deviation)
statements.add_validation_fun('reference_3', ['deviation'], _add_d_info)
statements.add_validation_fun('reference_3', ['deviate'], _remove_d_info)
filenames = []
#(name, rev, handle)
# where handle is (format, absfilename)
for (_, _, (_, filename)) in repos.get_modules_and_revisions(ctx):
filenames.append(filename)
modules = []
r = re.compile(r"^(.*?)(\@(\d{4}-\d{2}-\d{2}))?\.(yang|yin)$")
for filename in filenames:
f = filename
if filename.startswith('file://'):
f = filename[len('file://') - 1:]
try:
fd = open(f)
text = fd.read()
except IOError as ex:
err_msg = "error %s: %s\n" % (filename, str(ex))
logger.error(err_msg)
raise YdkGenException(err_msg)
m = r.search(filename)
ctx.yin_module_map = {}
if m is not None:
(name, _dummy, rev, _) = m.groups()
name = os.path.basename(name)
logger.debug('Parsing file %s format %s name %s revision %s',
filename, format, name, rev)
module = ctx.add_module(filename,
text,
format,
name,
rev,
expect_failure_error=False)
else:
module = ctx.add_module(filename, text)
if module is None:
raise YdkGenException('Could not add module ')
else:
modules.append(module)
# all the module have been added so get the context to validate
# call prevalidate before this and post validate after
ctx.validate()
def keyfun(e):
if e[0].ref == filenames[0]:
return 0
else:
return 1
ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
if len(filenames) > 0:
# first print error for the first filename given
ctx.errors.sort(key=keyfun)
error_messages = []
for (epos, etag, eargs) in ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
logger.warning('%s: %s\n' %
(str(epos), error.err_to_str(etag, eargs)))
else:
err_msg = '%s: %s\n' % (str(epos), error.err_to_str(etag, eargs))
logger.error(err_msg)
error_messages.append(err_msg)
if len(error_messages) > 0:
err_msg = '\n'.join(error_messages)
raise YdkGenException(err_msg)
return [m for m in modules if m.keyword == 'module']
def post_validate_ctx(self, ctx, modules):
nbr_option_specified = 0
if ctx.opts.generate_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.update_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.check_sid_file is not None:
nbr_option_specified += 1
if nbr_option_specified == 0:
return
if nbr_option_specified > 1:
sys.stderr.write(
"Invalid option, only one process on .sid file can be requested.\n"
)
return
if ctx.errors != []:
fatal_errors = 0
for (epos, etag, eargs) in ctx.errors:
elevel = error.err_level(etag)
if error.is_warning(elevel):
kind = "warning"
else:
kind = "error"
fatal_errors += 1
sys.stderr.write(str(epos) + ': %s: ' % kind + \
error.err_to_str(etag, eargs) + '\n')
if fatal_errors > 0:
sys.stderr.write(
"Invalid YANG module, .sid file processing aborted.\n")
return
sid_file = SidFile()
if ctx.opts.sid_registration_info:
sid_file.sid_registration_info = True
if ctx.opts.generate_sid_file is not None:
sid_file.range = ctx.opts.generate_sid_file
sid_file.is_consistent = False
sid_file.sid_file_created = True
if ctx.opts.update_sid_file is not None:
sid_file.input_file_name = ctx.opts.update_sid_file
if ctx.opts.check_sid_file is not None:
sid_file.input_file_name = ctx.opts.check_sid_file
sid_file.check_consistency = True
if not sid_file.sid_registration_info:
print("Checking consistency of '%s'" %
sid_file.input_file_name)
if ctx.opts.extra_sid_range is not None:
if ctx.opts.update_sid_file is not None:
sid_file.extra_range = ctx.opts.extra_sid_range
else:
sys.stderr.write(
"An extra SID range can be specified only during a .sid file update.\n"
)
return
if ctx.opts.list_sid:
sid_file.list_content = True
try:
sid_file.process_sid_file(modules[0])
except SidParcingError as e:
sys.stderr.write("ERROR, %s\n" % e.msg)
sys.exit(1)
except SidFileError as e:
sys.stderr.write("ERROR in '%s', %s\n" %
(sid_file.input_file_name, e.msg))
sys.exit(1)
except FileNotFoundError as e:
sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
sys.exit(1)
except json.decoder.JSONDecodeError as e:
sys.stderr.write(
"ERROR in '%s', line %d, column %d, %s\n" %
(sid_file.input_file_name, e.lineno, e.colno, e.msg))
sys.exit(1)
def post_validate_ctx(self, ctx, modules):
nbr_option_specified = 0
if ctx.opts.generate_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.update_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.check_sid_file is not None:
nbr_option_specified += 1
if nbr_option_specified == 0:
return
if nbr_option_specified > 1:
sys.stderr.write(
"Invalid option, only one process on .sid file can be requested.\n"
)
return
fatal_error = False
for _, etag, _ in ctx.errors:
if not error.is_warning(error.err_level(etag)):
fatal_error = True
if fatal_error or ctx.errors and ctx.opts.check_sid_file is not None:
sys.stderr.write("Invalid YANG module\n")
return
sid_file = SidFile()
if ctx.opts.sid_registration_info:
sid_file.sid_registration_info = True
if ctx.opts.generate_sid_file is not None:
sid_file.range = ctx.opts.generate_sid_file
sid_file.is_consistent = False
sid_file.sid_file_created = True
if ctx.opts.update_sid_file is not None:
sid_file.input_file_name = ctx.opts.update_sid_file
if ctx.opts.check_sid_file is not None:
sid_file.input_file_name = ctx.opts.check_sid_file
sid_file.check_consistency = True
if not sid_file.sid_registration_info:
print("Checking consistency of '%s'" %
sid_file.input_file_name)
if ctx.opts.extra_sid_range is not None:
if ctx.opts.update_sid_file is not None:
sid_file.extra_range = ctx.opts.extra_sid_range
else:
sys.stderr.write(
"An extra SID range can be specified only during a .sid file update.\n"
)
return
if ctx.opts.list_sid:
sid_file.list_content = True
try:
sid_file.process_sid_file(modules[0])
except SidParsingError as e:
sys.stderr.write("ERROR, %s\n" % e)
except SidFileError as e:
sys.stderr.write("ERROR in '%s', %s\n" %
(sid_file.input_file_name, e))
except EnvironmentError as e:
if e.errno == errno.ENOENT:
sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
else:
sys.stderr.write("ERROR, in file '%s' " % e.filename)
except JSONDecodeError as e:
sys.stderr.write("ERROR in '%s', %s\n" %
(sid_file.input_file_name, e))
except ValueError as e:
sys.stderr.write("ERROR in '%s', invalid JSON content\n" %
sid_file.input_file_name)
else:
sys.exit(0)
sys.exit(1)
def check_update(ctx, oldfilename, newmod):
oldpath = os.pathsep.join(ctx.opts.old_path)
olddir = os.path.dirname(oldfilename)
if olddir == '':
olddir = '.'
oldpath += os.pathsep + olddir
oldrepo = pyang.FileRepository(oldpath, use_env=False)
oldctx = pyang.Context(oldrepo)
oldctx.opts = ctx.opts
oldctx.lax_xpath_checks = ctx.lax_xpath_checks
oldctx.lax_quote_checks = ctx.lax_quote_checks
if ctx.opts.verbose:
print("Loading old modules from:")
for d in oldrepo.dirs:
print(" %s" % d)
print("")
for p in plugin.plugins:
p.setup_ctx(oldctx)
for oldfilename in [ctx.opts.check_update_from] + ctx.opts.old_deviation:
try:
fd = io.open(oldfilename, "r", encoding="utf-8")
text = fd.read()
except IOError as ex:
sys.stderr.write("error %s: %s\n" % (oldfilename, str(ex)))
sys.exit(1)
if oldfilename in ctx.opts.old_deviation:
oldctx.add_module(oldfilename, text)
else:
oldmod = oldctx.add_module(oldfilename, text)
ctx.errors.extend(oldctx.errors)
if oldmod is None:
return
for (epos, etag, eargs) in ctx.errors:
if (epos.ref in (newmod.pos.ref, oldmod.pos.ref) and
error.is_error(error.err_level(etag))):
return
if ctx.opts.verbose:
print("Loaded old modules:")
for x in oldrepo.get_modules_and_revisions(oldctx):
(m, r, (fmt, filename)) = x
print(" %s" % filename)
print("")
chk_modulename(oldmod, newmod, ctx)
chk_namespace(oldmod, newmod, ctx)
chk_revision(oldmod, newmod, ctx)
for olds in oldmod.search('feature'):
chk_feature(olds, newmod, ctx)
for olds in oldmod.search('identity'):
chk_identity(olds, newmod, ctx)
for olds in oldmod.search('typedef'):
chk_typedef(olds, newmod, ctx)
for olds in oldmod.search('grouping'):
chk_grouping(olds, newmod, ctx)
for olds in oldmod.search('rpc'):
chk_rpc(olds, newmod, ctx)
for olds in oldmod.search('notification'):
chk_notification(olds, newmod, ctx)
for olds in oldmod.search('extension'):
chk_extension(olds, newmod, ctx)
for olds in oldmod.search('augment'):
chk_augment(olds, newmod, ctx)
chk_i_children(oldmod, newmod, ctx)
context.errors.sort(key=keyfun)
# Parse out all the module names for error checking.
module_names = []
for module in modules:
module_names.append(module.arg)
for included_module in module.search('include'):
module_names.append(included_module.arg)
# Now actually do error parsing
for (epos, etag, eargs) in context.errors:
if (context.implicit_errors == False and hasattr(epos.top, 'i_modulename')
and epos.top.arg not in module_names
and epos.top.i_modulename not in module_names
and epos.ref not in model_filenames):
# this module was added implicitly (by import); skip this error
# the code includes submodules
continue
elevel = error.err_level(etag)
if error.is_warning(elevel):
kind = "error"
exit_code = 1
logging.error('%s: %s: %s', str(epos), kind, etag)
# Now actually do whatever we want.
parsed_modules = [recurse_children(module, module) for module in modules]
if len(parsed_modules) == 1:
parsed_modules = parsed_modules[0]
with open('testout.json', 'w') as testout_fd:
json.dump(parsed_modules, testout_fd)
def run():
usage = """%prog [options] [<filename>...]
Validates the YANG module in <filename> (or stdin), and all its dependencies."""
plugindirs = []
# check for --plugindir
idx = 1
while '--plugindir' in sys.argv[idx:]:
idx = idx + sys.argv[idx:].index('--plugindir')
plugindirs.append(sys.argv[idx + 1])
idx = idx + 1
plugin.init(plugindirs)
fmts = {}
xforms = {}
for p in plugin.plugins:
p.add_output_format(fmts)
p.add_transform(xforms)
optlist = [
# use capitalized versions of std options help and version
optparse.make_option("-h", "--help1",
action="help",
help="Show this help message and exit ha2"),
optparse.make_option("-v", "--version",
action="version",
help="Show version number and exit"),
optparse.make_option("-V", "--verbose",
action="store_true"),
optparse.make_option("-e", "--list-errors",
dest="list_errors",
action="store_true",
help="Print a listing of all error and warning " \
"codes and exit."),
optparse.make_option("--print-error-code",
dest="print_error_code",
action="store_true",
help="On errors, print the error code instead " \
"of the error message."),
optparse.make_option("-W",
dest="warnings",
action="append",
default=[],
metavar="WARNING",
help="If WARNING is 'error', treat all warnings " \
"as errors, except any listed WARNING. " \
"If WARNING is 'none', do not report any " \
"warnings."),
optparse.make_option("-E",
dest="errors",
action="append",
default=[],
metavar="WARNING",
help="Treat each WARNING as an error. For a " \
"list of warnings, use --list-errors."),
optparse.make_option("--ignore-error",
dest="ignore_error_tags",
action="append",
default=[],
metavar="ERROR",
help="Ignore ERROR. Use with care. For a " \
"list of errors, use --list-errors."),
optparse.make_option("--ignore-errors",
dest="ignore_errors",
action="store_true",
help="Ignore all errors. Use with care."),
optparse.make_option("--canonical",
dest="canonical",
action="store_true",
help="Validate the module(s) according to the " \
"canonical YANG order."),
optparse.make_option("--max-line-length",
type="int",
dest="max_line_len"),
optparse.make_option("--max-identifier-length",
type="int",
dest="max_identifier_len"),
optparse.make_option("-t", "--transform", dest="transforms",
default=[], action="append",
help="Apply transform TRANSFORM. Supported " \
"transforms are: " + ', '.join(list(
xforms.keys()))),
optparse.make_option("-f", "--format",
dest="format",
help="Convert to FORMAT. Supported formats " \
"are: " + ', '.join(list(fmts.keys()))),
optparse.make_option("-o", "--output",
dest="outfile",
help="Write the output to OUTFILE instead " \
"of stdout."),
optparse.make_option("-F", "--features",
metavar="FEATURES",
dest="features",
default=[],
action="append",
help="Features to support, default all. " \
"<modname>:[<feature>,]*"),
optparse.make_option("", "--max-status",
metavar="MAXSTATUS",
dest="max_status",
help="Max status to support, one of: " \
"current, deprecated, obsolete"),
optparse.make_option("", "--deviation-module",
metavar="DEVIATION",
dest="deviations",
default=[],
action="append",
help="Deviation module"),
optparse.make_option("-p", "--path",
dest="path",
default=[],
action="append",
help=os.pathsep + "-separated search path for yin"
" and yang modules"),
optparse.make_option("--plugindir",
dest="plugindir",
help="Load pyang plugins from PLUGINDIR"),
optparse.make_option("--strict",
dest="strict",
action="store_true",
help="Force strict YANG compliance."),
optparse.make_option("--lax-quote-checks",
dest="lax_quote_checks",
action="store_true",
help="Lax check of backslash in quoted strings."),
optparse.make_option("--lax-xpath-checks",
dest="lax_xpath_checks",
action="store_true",
help="Lax check of XPath expressions."),
optparse.make_option("--trim-yin",
dest="trim_yin",
action="store_true",
help="In YIN input modules, trim whitespace "
"in textual arguments."),
optparse.make_option("-L", "--hello",
dest="hello",
action="store_true",
help="Filename of a server's hello message is "
"given instead of module filename(s)."),
optparse.make_option("--keep-comments",
dest="keep_comments",
action="store_true",
help="Pyang will not discard comments; \
has effect if the output plugin can \
handle comments." ),
optparse.make_option("--no-path-recurse",
dest="no_path_recurse",
action="store_true",
help="Do not recurse into directories in the \
yang path." ),
]
optparser = optparse.OptionParser(usage, add_help_option=False)
optparser.version = '%prog ' + pyang.__version__
optparser.add_options(optlist)
for p in plugin.plugins:
p.add_opts(optparser)
(o, args) = optparser.parse_args()
if o.list_errors == True:
for tag in error.error_codes:
(level, fmt) = error.error_codes[tag]
if error.is_warning(level):
print("Warning: %s" % tag)
elif error.allow_warning(level):
print("Minor Error: %s" % tag)
else:
print("Error: %s" % tag)
print("Message: %s" % fmt)
print("")
sys.exit(0)
if o.outfile != None and o.format == None:
sys.stderr.write("no format specified\n")
sys.exit(1)
# patch the error spec so that -W errors are treated as warnings
for w in o.warnings:
if w in error.error_codes:
(level, wstr) = error.error_codes[w]
if error.allow_warning(level):
error.error_codes[w] = (4, wstr)
filenames = args
# Parse hello if present
if o.hello:
if len(filenames) > 1:
sys.stderr.write("multiple hello files given\n")
sys.exit(1)
if filenames:
try:
fd = open(filenames[0], "rb")
except IOError as ex:
sys.stderr.write("error %s: %s\n" % (filenames[0], str(ex)))
sys.exit(1)
elif sys.version < "3":
fd = sys.stdin
else:
fd = sys.stdin.buffer
hel = hello.HelloParser().parse(fd)
path = os.pathsep.join(o.path)
# add standard search path
if len(o.path) == 0:
path = "."
else:
path += os.pathsep + "."
repos = pyang.FileRepository(path,
no_path_recurse=o.no_path_recurse,
verbose=o.verbose)
ctx = pyang.Context(repos)
ctx.opts = o
ctx.canonical = o.canonical
ctx.max_line_len = o.max_line_len
ctx.max_identifier_len = o.max_identifier_len
ctx.trim_yin = o.trim_yin
ctx.lax_xpath_checks = o.lax_xpath_checks
ctx.lax_quote_checks = o.lax_quote_checks
ctx.strict = o.strict
ctx.max_status = o.max_status
# make a map of features to support, per module
if o.hello:
for (mn, rev) in hel.yang_modules():
ctx.features[mn] = hel.get_features(mn)
for f in ctx.opts.features:
(modulename, features) = parse_features_string(f)
ctx.features[modulename] = features
for p in plugin.plugins:
p.setup_ctx(ctx)
xform_objs = []
for transform in o.transforms:
if transform not in xforms:
sys.stderr.write("unsupported transform '%s'\n" % transform)
else:
xform_obj = xforms[transform]
xform_obj.setup_xform(ctx)
xform_objs.append(xform_obj)
if len(xform_objs) != len(o.transforms):
sys.exit(1)
if o.format != None:
if o.format not in fmts:
sys.stderr.write("unsupported format '%s'\n" % o.format)
sys.exit(1)
emit_obj = fmts[o.format]
if o.keep_comments and emit_obj.handle_comments:
ctx.keep_comments = True
emit_obj.setup_fmt(ctx)
else:
emit_obj = None
xform_and_emit_objs = xform_objs[:]
if emit_obj is not None:
xform_and_emit_objs.append(emit_obj)
for p in plugin.plugins:
p.pre_load_modules(ctx)
exit_code = 0
modules = []
if o.hello:
ctx.capabilities = hel.registered_capabilities()
for (mn, rev) in hel.yang_modules():
mod = ctx.search_module(0, mn, rev)
if mod is None:
emarg = mn
if rev: emarg += "@" + rev
sys.stderr.write(
"module '%s' specified in hello not found.\n" % emarg)
sys.exit(1)
modules.append(mod)
else:
if len(filenames) == 0:
text = sys.stdin.read()
module = ctx.add_module('<stdin>', text)
if module is None:
exit_code = 1
else:
modules.append(module)
if (len(filenames) > 1 and emit_obj is not None
and not emit_obj.multiple_modules):
sys.stderr.write("too many files to convert\n")
sys.exit(1)
for filename in filenames:
try:
fd = io.open(filename, "r", encoding="utf-8")
text = fd.read()
if o.verbose:
util.report_file_read(filename, "(CL)")
except IOError as ex:
sys.stderr.write("error %s: %s\n" % (filename, str(ex)))
sys.exit(1)
except UnicodeDecodeError as ex:
s = str(ex).replace('utf-8', 'utf8')
sys.stderr.write("%s: unicode error: %s\n" % (filename, s))
sys.exit(1)
m = syntax.re_filename.search(filename)
ctx.yin_module_map = {}
if m is not None:
(name, rev, format) = m.groups()
name = os.path.basename(name)
module = ctx.add_module(filename,
text,
format,
name,
rev,
expect_failure_error=False)
else:
module = ctx.add_module(filename, text)
if module is None:
exit_code = 1
else:
modules.append(module)
modulenames = []
for m in modules:
modulenames.append(m.arg)
for s in m.search('include'):
modulenames.append(s.arg)
# apply deviations
for filename in ctx.opts.deviations:
try:
fd = io.open(filename, "r", encoding="utf-8")
text = fd.read()
except IOError as ex:
sys.stderr.write("error %s: %s\n" % (filename, str(ex)))
sys.exit(1)
except UnicodeDecodeError as ex:
s = str(ex).replace('utf-8', 'utf8')
sys.stderr.write("%s: unicode error: %s\n" % (filename, s))
sys.exit(1)
m = ctx.add_module(filename, text)
if m is not None:
ctx.deviation_modules.append(m)
for p in plugin.plugins:
p.pre_validate_ctx(ctx, modules)
if len(xform_and_emit_objs) > 0 and len(modules) > 0:
for obj in xform_and_emit_objs:
obj.pre_validate(ctx, modules)
ctx.validate()
for m in modules:
m.prune()
# transform modules
if len(xform_objs) > 0 and len(modules) > 0:
for xform_obj in xform_objs:
try:
if not xform_obj.transform(ctx, modules):
for module in modules:
module.i_is_validated = False
statements.validate_module(ctx, module)
except error.TransformError as e:
if e.msg != "":
sys.stderr.write(e.msg + '\n')
sys.exit(e.exit_code)
# verify the given features
for m in modules:
if m.arg in ctx.features:
for f in ctx.features[m.arg]:
if f not in m.i_features:
sys.stderr.write("unknown feature %s in module %s\n" %
(f, m.arg))
sys.exit(1)
if len(xform_and_emit_objs) > 0 and len(modules) > 0:
for obj in xform_and_emit_objs:
obj.post_validate(ctx, modules)
for p in plugin.plugins:
p.post_validate_ctx(ctx, modules)
def keyfun(e):
if e[0].ref == filenames[0]:
return 0
else:
return 1
ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
if len(filenames) > 0:
# first print error for the first filename given
ctx.errors.sort(key=keyfun)
if o.ignore_errors:
ctx.errors = []
for (epos, etag, eargs) in ctx.errors:
if etag in o.ignore_error_tags:
continue
if (ctx.implicit_errors == False and hasattr(epos.top, 'i_modulename')
and epos.top.arg not in modulenames
and epos.top.i_modulename not in modulenames
and epos.ref not in filenames):
# this module was added implicitly (by import); skip this error
# the code includes submodules
continue
elevel = error.err_level(etag)
if error.is_warning(elevel) and etag not in o.errors:
kind = "warning"
if 'error' in o.warnings and etag not in o.warnings:
kind = "error"
exit_code = 1
elif 'none' in o.warnings:
continue
else:
kind = "error"
exit_code = 1
if o.print_error_code == True:
sys.stderr.write(str(epos) + ': %s: %s\n' % (kind, etag))
else:
sys.stderr.write(str(epos) + ': %s: ' % kind + \
error.err_to_str(etag, eargs) + '\n')
if emit_obj is not None and len(modules) > 0:
tmpfile = None
if o.outfile == None:
if sys.version < '3':
fd = codecs.getwriter('utf8')(sys.stdout)
else:
fd = sys.stdout
else:
tmpfile = o.outfile + ".tmp"
if sys.version < '3':
fd = codecs.open(tmpfile, "w+", encoding="utf-8")
else:
fd = io.open(tmpfile, "w+", encoding="utf-8")
try:
# 调用具体模块的实现
emit_obj.emit(ctx, modules, fd)
except error.EmitError as e:
if e.msg != "":
sys.stderr.write(e.msg + '\n')
if tmpfile != None:
fd.close()
os.remove(tmpfile)
sys.exit(e.exit_code)
except:
if tmpfile != None:
fd.close()
os.remove(tmpfile)
raise
if tmpfile != None:
fd.close()
os.rename(tmpfile, o.outfile)
sys.exit(exit_code)
def post_validate_ctx(self, ctx, modules):
nbr_option_specified = 0
if ctx.opts.generate_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.update_sid_file is not None:
nbr_option_specified += 1
if ctx.opts.check_sid_file is not None:
nbr_option_specified += 1
if nbr_option_specified == 0:
return
if nbr_option_specified > 1:
sys.stderr.write("Invalid option, only one process on .sid file can be requested.\n")
return
fatal_error = False
for (epos, etag, eargs) in ctx.errors:
if not error.is_warning(error.err_level(etag)):
fatal_error = True
if fatal_error or (ctx.errors != [] and ctx.opts.check_sid_file is not None):
sys.stderr.write("Invalid YANG module\n")
return
sid_file = SidFile()
if ctx.opts.sid_registration_info:
sid_file.sid_registration_info = True
if ctx.opts.generate_sid_file is not None:
sid_file.range = ctx.opts.generate_sid_file
sid_file.is_consistent = False
sid_file.sid_file_created = True
if ctx.opts.update_sid_file is not None:
sid_file.input_file_name = ctx.opts.update_sid_file
if ctx.opts.check_sid_file is not None:
sid_file.input_file_name = ctx.opts.check_sid_file
sid_file.check_consistency = True
if not sid_file.sid_registration_info:
print("Checking consistency of '%s'" % sid_file.input_file_name)
if ctx.opts.extra_sid_range is not None:
if ctx.opts.update_sid_file is not None:
sid_file.extra_range = ctx.opts.extra_sid_range
else:
sys.stderr.write("An extra SID range can be specified only during a .sid file update.\n")
return
if ctx.opts.list_sid:
sid_file.list_content = True
try:
sid_file.process_sid_file(modules[0])
except SidParsingError as e:
sys.stderr.write("ERROR, %s\n" % e.msg)
sys.exit(1)
except SidFileError as e:
sys.stderr.write("ERROR in '%s', %s\n" % (sid_file.input_file_name, e.msg))
sys.exit(1)
except EnvironmentError as e:
if e.errno == errno.ENOENT:
sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
else:
sys.stderr.write("ERROR, in file '%s' " % e.filename)
sys.exit(1)
except ValueError as e:
if hasattr(e, "lineno") and sid_file.input_file_name != "*****@*****.**":
# Present only in python 3.5 and later, except json.decoder.JSONDecodeError
sys.stderr.write("ERROR in '%s', line %d, column %d, %s\n" % (sid_file.input_file_name, e.lineno, e.colno, e.msg))
else:
sys.stderr.write("ERROR in '%s', invalid JSON content\n" % sid_file.input_file_name)
sys.exit(1)