Example #1
0
def emit_json_xsl(modules, ctx, fd):
    """Main control function.

    Set up the top-level parts of the stylesheet, the process
    recursively all nodes in all data trees, and finally emit the
    serialized stylesheet.
    """
    for (epos, etag, eargs) in ctx.errors:
        if error.is_error(error.err_level(etag)):
            raise error.EmitError("JSONXSL plugin needs a valid module")
    tree = ET.ElementTree(ss)
    ET.SubElement(ss, "output", method="text")
    xsltdir = os.environ.get("PYANG_XSLT_DIR", "/usr/local/share/yang/xslt")
    ET.SubElement(ss, "include", href=xsltdir + "/jsonxsl-templates.xsl")
    ET.SubElement(ss, "strip-space", elements="*")
    nsmap = ET.SubElement(ss, "template", name="nsuri-to-module")
    ET.SubElement(nsmap, "param", name="uri")
    choo = ET.SubElement(nsmap, "choose")
    for module in modules:
        ns_uri = module.search_one("namespace").arg
        ss.attrib["xmlns:" + module.i_prefix] = ns_uri
        when = ET.SubElement(choo, "when", test="$uri='" + ns_uri + "'")
        xsl_text(module.i_modulename, when)
        process_module(module)
    tree.write(fd, encoding="utf-8", xml_declaration=True)
Example #2
0
    def _validate_pyang_modules(self, filenames):
        # all the module have been added so get the context to validate
        # call prevalidate before this and post validate after
        self.ctx.validate()

        def keyfun(e):
            if e[0].ref == filenames[0]:
                return 0
            else:
                return 1

        self.ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
        if len(filenames) > 0:
            # first print error for the first filename given
            self.ctx.errors.sort(key=keyfun)

        error_messages = []
        for (epos, etag, eargs) in self.ctx.errors:
            elevel = error.err_level(etag)
            if error.is_warning(elevel):
                logger.warning('%s: %s\n' %
                               (str(epos), error.err_to_str(etag, eargs)))
            else:
                err_msg = '%s: %s\n' % (str(epos), error.err_to_str(
                    etag, eargs))
                logger.error(err_msg)
                error_messages.append(err_msg)

        if len(error_messages) > 0:
            err_msg = '\n'.join(error_messages)
            raise YdkGenException(err_msg)
Example #3
0
    def _validate_pyang_modules(self, filenames):
        # all the module have been added so get the context to validate
        # call prevalidate before this and post validate after
        self.ctx.validate()

        def keyfun(e):
            if e[0].ref == filenames[0]:
                return 0
            else:
                return 1

        self.ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
        if len(filenames) > 0:
            # first print error for the first filename given
            self.ctx.errors.sort(key=keyfun)

        error_messages = []
        for (epos, etag, eargs) in self.ctx.errors:
            elevel = error.err_level(etag)
            if error.is_warning(elevel):
                logger.warning('%s: %s\n' %
                               (str(epos), error.err_to_str(etag, eargs)))
            else:
                err_msg = '%s: %s\n' % (str(epos), error.err_to_str(etag, eargs))
                logger.error(err_msg)
                error_messages.append(err_msg)

        if len(error_messages) > 0:
            err_msg = '\n'.join(error_messages)
            raise YdkGenException('''\nError occured: "%s". \nThe models supplied to the YDK generator are invalid. Please make sure the models are valid by compiling the models together using pyang. Please run "pyang *" in the models directory, make sure there are no errors and then try running the generator again. If there are model errors, please fix the errors by editing the model, contacting the model owner or deleting the model from the list of models to generate the YDK bindings for.'''%err_msg)
Example #4
0
 def emit(self, ctx, modules, fd):
     modulenames = [m.arg for m in modules]
     if not ctx.opts.ignore_errors:
         for (epos, etag, eargs) in ctx.errors:
             if (epos.top.arg in modulenames
                     and error.is_error(error.err_level(etag))):
                 raise error.EmitError("%s contains errors" % epos.top.arg)
     emit_docs(ctx, modules, fd)
Example #5
0
 def emit(self, ctx, modules, fd):
     # cannot do this unless everything is ok for our module
     modulenames = [m.arg for m in modules]
     for (epos, etag, eargs) in ctx.errors:
         if (epos.top.arg in modulenames
                 and error.is_error(error.err_level(etag))):
             raise error.EmitError("%s contains errors" % epos.top.arg)
     emit_depend(ctx, modules, fd)
    def emit(self, ctx, modules, fd):
        """Main control function.

        Set up the top-level parts of the sample document, then process
        recursively all nodes in all data trees, and finally emit the
        sample XML document.
        """
        if ctx.opts.sample_path is not None:
            path = ctx.opts.sample_path.split('/')
            if path[0] == '':
                path = path[1:]
        else:
            path = []

        for (epos, etag, eargs) in ctx.errors:
            if error.is_error(error.err_level(etag)):
                raise error.EmitError(
                    "sample-xml-skeleton plugin needs a valid module")
        self.doctype = ctx.opts.doctype
        if self.doctype not in ("config", "data"):
            raise error.EmitError("Unsupported document type: %s" %
                                  self.doctype)
        self.annots = ctx.opts.sample_annots
        self.defaults = ctx.opts.sample_defaults
        self.node_handler = {
            "container": self.container,
            "leaf": self.leaf,
            "anyxml": self.anyxml,
            "choice": self.process_children,
            "case": self.process_children,
            "list": self.list,
            "leaf-list": self.leaf_list,
            "rpc": self.ignore,
            "action": self.ignore,
            "notification": self.ignore
        }
        self.ns_uri = {}
        for yam in modules:
            self.ns_uri[yam] = yam.search_one("namespace").arg
        self.top = etree.Element(
            self.doctype, {"xmlns": "urn:ietf:params:xml:ns:netconf:base:1.0"})
        tree = etree.ElementTree(self.top)
        for yam in modules:
            self.process_children(yam, self.top, None, path)
        if sys.version > "3":
            fd.write(
                str(
                    etree.tostring(tree,
                                   pretty_print=True,
                                   encoding="UTF-8",
                                   xml_declaration=True), "UTF-8"))
        elif sys.version > "2.7":
            tree.write(fd,
                       encoding="UTF-8",
                       pretty_print=True,
                       xml_declaration=True)
        else:
            tree.write(fd, pretty_print=True, encoding="UTF-8")
 def emit(self, ctx, modules, fd):
     # cannot do this unless everything is ok for our module
     modulenames = [m.arg for m in modules]
     for (epos, etag, eargs) in ctx.errors:
         if ((epos.top is None or epos.top.arg in modulenames)
                 and error.is_error(error.err_level(etag))):
             raise error.EmitError("%s contains more fundamental errors "\
                 "than the pattern statements" % epos.top.arg)
     emit_clean_pattern(ctx, modules, fd)
Example #8
0
def emit_dsdl(ctx, modules, fd):
    for (epos, etag, eargs) in ctx.errors:
        if error.is_error(error.err_level(etag)):
            raise error.EmitError("DSDL translation needs a valid module")
    schema = HybridDSDLSchema().from_modules(modules,
                                  ctx.opts.dsdl_no_dublin_core,
                                  ctx.opts.dsdl_no_documentation,
                                  ctx.opts.dsdl_record_defs, debug=0)
    fd.write(schema.serialize())
Example #9
0
    def emit(self, ctx, modules, fd):
        # Require error-free modules
        modulenames = [m.arg for m in modules]
        for (epos, etag, eargs) in ctx.errors:
            if (epos.top.arg in modulenames
                    and error.is_error(error.err_level(etag))):
                raise error.EmitError("%s contains errors" % epos.top.arg)

        emit_dot(ctx, modules, fd)
Example #10
0
                        def __print_pyang_output(ctx):
                            err = ''
                            for (epos, etag, eargs) in ctx.errors:
                                elevel = error.err_level(etag)
                                if error.is_warning(elevel):
                                    kind = "warning"
                                else:
                                    kind = "error"

                                err += str(epos) + ': %s: ' % kind + \
                                       error.err_to_str(etag, eargs) + '\n'
                            return err
    def __print_pyang_output(self):
        err = ''
        out = ''
        for (epos, etag, eargs) in self.__ctx.errors:
            elevel = error.err_level(etag)
            if error.is_warning(elevel):
                kind = 'warning'
            else:
                kind = 'error'

            err += '{}: {}: {}\n'.format(epos, kind, error.err_to_str(etag, eargs))
        return err, out
Example #12
0
def emit_cts(ctx, module, fd):
    # No errors are allowed
    for (epos, etag, eargs) in ctx.errors:
        if (epos.top_name == module.arg
                and error.is_error(error.err_level(etag))):
            raise error.EmitError("CTS translation needs a valid module")
    schema = ConceptualTreeSchema().from_modules((module, ),
                                                 ctx.opts.cts_no_dublin_core,
                                                 ctx.opts.cts_no_documentation,
                                                 ctx.opts.cts_record_defs,
                                                 debug=0)
    fd.write(schema.serialize())
Example #13
0
def check_update(ctx, newmod):
    oldpath = os.pathsep.join(ctx.opts.old_path)
    olddir = os.path.dirname(ctx.opts.check_update_from)
    if olddir == '':
        olddir = '.'
    oldpath += os.pathsep + olddir
    oldrepo = repository.FileRepository(oldpath, use_env=False)
    oldctx = context.Context(oldrepo)
    oldctx.opts = ctx.opts
    oldctx.lax_xpath_checks = ctx.lax_xpath_checks
    oldctx.lax_quote_checks = ctx.lax_quote_checks

    if ctx.opts.verbose:
        print("Loading old modules from:")
        for d in oldrepo.dirs:
            print("  %s" % d)
        print("")

    for p in plugin.plugins:
        p.setup_ctx(oldctx)

    for oldfilename in [ctx.opts.check_update_from] + ctx.opts.old_deviation:
        try:
            fd = io.open(oldfilename, "r", encoding="utf-8")
            text = fd.read()
        except IOError as ex:
            sys.stderr.write("error %s: %s\n" % (oldfilename, ex))
            sys.exit(1)
        if oldfilename in ctx.opts.old_deviation:
            oldctx.add_module(oldfilename, text)
        else:
            oldmod = oldctx.add_module(oldfilename, text)
    oldctx.validate()

    ctx.errors.extend(oldctx.errors)

    if oldmod is None:
        return

    for epos, etag, eargs in ctx.errors:
        if (epos.ref in (newmod.pos.ref, oldmod.pos.ref)
                and error.is_error(error.err_level(etag))):
            return

    if ctx.opts.verbose:
        print("Loaded old modules:")
        for x in oldrepo.get_modules_and_revisions(oldctx):
            (m, r, (fmt, filename)) = x
            print("  %s" % filename)
        print("")

    chk_module(ctx, oldmod, newmod)
Example #14
0
    def emit(self, ctx, modules, fd):
        for (epos, etag, eargs) in ctx.errors:
            if ((epos.top is None or epos.top.arg in self.mods)
                    and error.is_error(error.err_level(etag))):
                self.fatal("%s contains errors" % epos.top.arg)

        if ctx.opts.uml_pages_layout is not None:
            if re.match('[0-9]x[0-9]', ctx.opts.uml_pages_layout) is None:
                self.fatal(
                    "Illegal page split option %s, should be [0-9]x[0-9], example 2x2"
                    % ctx.opts.uml_pages_layout)

        umldoc = uml_emitter(ctx)
        umldoc.emit(modules, fd)
Example #15
0
    def emit(self, ctx, module, fd):
        # cannot do XSD unless everything is ok for our module
        for (epos, etag, eargs) in ctx.errors:
            if (epos.top_name == module.arg
                    and error.is_error(error.err_level(etag))):
                raise error.EmitError("XSD translation needs a valid module")
        # we also need to have all other modules found
        for pre in module.i_prefixes:
            (modname, revision) = module.i_prefixes[pre]
            mod = statements.modulename_to_module(module, modname, revision)
            if mod == None:
                raise error.EmitError("cannot find module %s, needed by XSD"
                                      " translation" % modname)

        emit_xsd(ctx, module, fd)
Example #16
0
 def emit(self, ctx, modules, fd):
   modulenames = [m.arg for m in modules]
   if not ctx.opts.ignore_errors:
     for (epos, etag, eargs) in ctx.errors:
       #
       #  When a module has not yet been parsed then the top.arg does
       #  not exist. This can be the case when an error is created early
       #  in the parsing process.
       #
       if not hasattr(epos.top, "arg"):
         raise error.EmitError("%s contains errors, and was not parsed"
             % (epos.ref))
       if (epos.top.arg in modulenames and
                 error.is_error(error.err_level(etag))):
           raise error.EmitError("%s contains errors" % epos.top.arg)
   emit_paths(ctx, modules, fd)
Example #17
0
    def emit(self, ctx, modules, fd):
        """Main control function.
        """
        for (epos, etag, eargs) in ctx.errors:
            if error.is_error(error.err_level(etag)):
                raise error.EmitError("sample-json-skeleton plugin needs a valid module")
        tree = {}
        self.defaults = ctx.opts.sample_defaults
        self.doctype = ctx.opts.doctype
        if self.doctype not in ("config", "data"):
            raise error.EmitError("Unsupported document type: %s" %
                                  self.doctype)

        for module in modules:
            self.process_children(module, tree, None)
        json.dump(tree, fd, indent=4)
Example #18
0
 def emit(self, ctx, modules, fd):
     """Main control function.
     """
     for (epos, etag, eargs) in ctx.errors:
         if error.is_error(error.err_level(etag)):
             raise error.EmitError("JTOX plugin needs a valid module")
     tree = {}
     mods = {}
     annots = {}
     for m, p in unique_prefixes(ctx).items():
         mods[m.i_modulename] = [p, m.search_one("namespace").arg]
     for module in modules:
         for ann in module.search(("ietf-yang-metadata", "annotation")):
             typ = ann.search_one("type")
             annots[module.arg + ":" + ann.arg] = ("string" if typ is None
                                                   else self.base_type(typ))
     for module in modules:
         self.process_children(module, tree, None)
     json.dump({"modules": mods, "tree": tree, "annotations": annots}, fd)
Example #19
0
    def emit(self, ctx, modules, fd):
        """Main control function.

        Set up the top-level parts of the stylesheet, then process
        recursively all nodes in all data trees, and finally emit the
        serialized stylesheet.
        """
        for (epos, etag, eargs) in ctx.errors:
            if error.is_error(error.err_level(etag)):
                raise error.EmitError("JSONXSL plugin needs a valid module")
        self.real_prefix = unique_prefixes(ctx)
        self.top_names = []
        for m in modules:
            self.top_names.extend([
                c.arg for c in m.i_children
                if c.keyword not in ("rpc", "notification")
            ])
        tree = ET.ElementTree(ss)
        ET.SubElement(ss, "output", method="text")
        xsltdir = os.environ.get("PYANG_XSLT_DIR",
                                 "/usr/local/share/yang/xslt")
        ET.SubElement(ss, "include", href=xsltdir + "/jsonxsl-templates.xsl")
        ET.SubElement(ss, "strip-space", elements="*")
        nsmap = ET.SubElement(ss, "template", name="nsuri-to-module")
        ET.SubElement(nsmap, "param", name="uri")
        choo = ET.SubElement(nsmap, "choose")
        for module in self.real_prefix.keys():
            ns_uri = module.search_one("namespace").arg
            ss.attrib["xmlns:" + self.real_prefix[module]] = ns_uri
            when = ET.SubElement(choo, "when", test="$uri='" + ns_uri + "'")
            self.xsl_text(module.i_modulename, when)
            self.process_module(module)
        if sys.version > "3":
            tree.write(fd, encoding="unicode", xml_declaration=True)
        elif sys.version > "2.7":
            tree.write(fd, encoding="UTF-8", xml_declaration=True)
        else:
            tree.write(fd, encoding="UTF-8")
Example #20
0
def emit_jtox(modules, ctx, fd):
    """Main control function.
    """
    for (epos, etag, eargs) in ctx.errors:
        if error.is_error(error.err_level(etag)):
            raise error.EmitError("JTOX plugin needs a valid module")
    tree = {}
    prefixes = []
    def unique_prefix(p):
        """Disambiguate the module prefix."""
        suffix = 0
        while p == "nc" or p in prefixes:
            p += "%d" % suffix
            suffix += 1
        return p
    for module in modules:
        uri = module.search_one("namespace").arg
        prf = unique_prefix(module.i_prefix)
        prefixes.append(prf)
        mods[module.i_modulename] = [prf, uri]
    for module in modules:
        process_children(module, tree)
    json.dump({"modules": mods, "tree": tree}, fd)
Example #21
0
    def emit(self, ctx, modules, fd):
        """Generates Java classes from the YANG module supplied to pyang.

        The generated classes are placed in the directory specified by the '-d'
        or '--jnc-output' flag, or in "gen" if no such flag was provided,
        using the 'directory' attribute of ctx. If there are existing files
        in the output directory with the same name as the generated classes,
        they are silently overwritten.

        ctx     -- Context used to get output directory, verbosity mode, error
                   handling policy (the ignore attribute) and whether or not a
                   schema file should be generated.
        modules -- A list containing a module statement parsed from the YANG
                   module supplied to pyang.
        fd      -- File descriptor (ignored).

        """
        if ctx.opts.debug or ctx.opts.verbose:
            print('JNC plugin starting')
        if not ctx.opts.ignore:
            for (epos, etag, _) in ctx.errors:
                if (error.is_error(error.err_level(etag)) and etag
                        in ('MODULE_NOT_FOUND', 'MODULE_NOT_FOUND_REV')):
                    self.fatal("%s contains errors" % epos.top.arg)
                if (etag in ('TYPE_NOT_FOUND', 'FEATURE_NOT_FOUND',
                             'IDENTITY_NOT_FOUND', 'GROUPING_NOT_FOUND')):
                    util.print_warning(
                        msg=(etag.lower() + ', generated class ' +
                             'hierarchy might be incomplete.'),
                        key=etag)
                else:
                    util.print_warning(msg=(etag.lower() + ', aborting.'),
                                       key=etag)
                    self.fatal("%s contains errors" % epos.top.arg)

        # Sweep, adding included and imported modules, until no change
        module_set = set(modules)
        num_modules = 0
        while num_modules != len(module_set):
            num_modules = len(module_set)
            for module in list(module_set):
                imported = map(lambda x: x.arg, util.search(module, 'import'))
                included = map(lambda x: x.arg, util.search(module, 'include'))
                for (module_stmt, rev) in self.ctx.modules:
                    if module_stmt in chain(imported, included):
                        module_set.add(self.ctx.modules[(module_stmt, rev)])

        # Generate files from main modules
        for module in filter(lambda s: s.keyword == 'module', module_set):
            self.generate_from(module)

        # Generate files from augmented modules
        for aug_module in context.augmented_modules.values():
            self.generate_from(aug_module)

        # Print debug messages saying that we're done.
        if ctx.opts.debug or ctx.opts.verbose:
            if not self.ctx.opts.no_classes:
                print('Java classes generation COMPLETE.')
            if not self.ctx.opts.no_schema:
                print('Schema generation COMPLETE.')
Example #22
0
def _parse_and_return_modules(resolved_model_dir):
    """ Use pyang to parse the files and get a list of modules.

        :param str resolved_model_dir The directory where all models to be compiled are found.
        :raise YdkGenException If there was a problem parsing the modules
    """
    repos = pyang.FileRepository(resolved_model_dir, False)
    ctx = pyang.Context(repos)

    statements.add_validation_fun('reference_3', ['deviation'],
                                  _add_i_deviation)
    statements.add_validation_fun('reference_3', ['deviation'], _add_d_info)
    statements.add_validation_fun('reference_3', ['deviate'], _remove_d_info)

    filenames = []

    #(name, rev, handle)
    # where handle is (format, absfilename)
    for (_, _, (_, filename)) in repos.get_modules_and_revisions(ctx):
        filenames.append(filename)

    modules = []

    r = re.compile(r"^(.*?)(\@(\d{4}-\d{2}-\d{2}))?\.(yang|yin)$")
    for filename in filenames:
        f = filename
        if filename.startswith('file://'):
            f = filename[len('file://') - 1:]
        try:
            fd = open(f)
            text = fd.read()
        except IOError as ex:
            err_msg = "error %s: %s\n" % (filename, str(ex))
            logger.error(err_msg)
            raise YdkGenException(err_msg)

        m = r.search(filename)
        ctx.yin_module_map = {}
        if m is not None:
            (name, _dummy, rev, _) = m.groups()
            name = os.path.basename(name)
            logger.debug('Parsing file %s format %s name %s revision %s',
                         filename, format, name, rev)
            module = ctx.add_module(filename,
                                    text,
                                    format,
                                    name,
                                    rev,
                                    expect_failure_error=False)
        else:
            module = ctx.add_module(filename, text)
        if module is None:
            raise YdkGenException('Could not add module ')
        else:
            modules.append(module)

    # all the module have been added so get the context to validate
    # call prevalidate before this and post validate after
    ctx.validate()

    def keyfun(e):
        if e[0].ref == filenames[0]:
            return 0
        else:
            return 1

    ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
    if len(filenames) > 0:
        # first print error for the first filename given
        ctx.errors.sort(key=keyfun)

    error_messages = []
    for (epos, etag, eargs) in ctx.errors:

        elevel = error.err_level(etag)
        if error.is_warning(elevel):
            logger.warning('%s: %s\n' %
                           (str(epos), error.err_to_str(etag, eargs)))
        else:
            err_msg = '%s: %s\n' % (str(epos), error.err_to_str(etag, eargs))
            logger.error(err_msg)
            error_messages.append(err_msg)

    if len(error_messages) > 0:
        err_msg = '\n'.join(error_messages)
        raise YdkGenException(err_msg)

    return [m for m in modules if m.keyword == 'module']
Example #23
0
File: sid.py Project: mcr/yang-cbor
    def post_validate_ctx(self, ctx, modules):
        nbr_option_specified = 0
        if ctx.opts.generate_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.update_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.check_sid_file is not None:
            nbr_option_specified += 1
        if nbr_option_specified == 0:
            return
        if nbr_option_specified > 1:
            sys.stderr.write(
                "Invalid option, only one process on .sid file can be requested.\n"
            )
            return

        if ctx.errors != []:
            fatal_errors = 0
            for (epos, etag, eargs) in ctx.errors:
                elevel = error.err_level(etag)
                if error.is_warning(elevel):
                    kind = "warning"
                else:
                    kind = "error"
                    fatal_errors += 1
                sys.stderr.write(str(epos) + ': %s: ' % kind + \
                                 error.err_to_str(etag, eargs) + '\n')

            if fatal_errors > 0:
                sys.stderr.write(
                    "Invalid YANG module, .sid file processing aborted.\n")
                return

        sid_file = SidFile()

        if ctx.opts.sid_registration_info:
            sid_file.sid_registration_info = True

        if ctx.opts.generate_sid_file is not None:
            sid_file.range = ctx.opts.generate_sid_file
            sid_file.is_consistent = False
            sid_file.sid_file_created = True

        if ctx.opts.update_sid_file is not None:
            sid_file.input_file_name = ctx.opts.update_sid_file

        if ctx.opts.check_sid_file is not None:
            sid_file.input_file_name = ctx.opts.check_sid_file
            sid_file.check_consistency = True
            if not sid_file.sid_registration_info:
                print("Checking consistency of '%s'" %
                      sid_file.input_file_name)

        if ctx.opts.extra_sid_range is not None:
            if ctx.opts.update_sid_file is not None:
                sid_file.extra_range = ctx.opts.extra_sid_range
            else:
                sys.stderr.write(
                    "An extra SID range can be specified only during a .sid file update.\n"
                )
                return

        if ctx.opts.list_sid:
            sid_file.list_content = True

        try:
            sid_file.process_sid_file(modules[0])

        except SidParcingError as e:
            sys.stderr.write("ERROR, %s\n" % e.msg)
            sys.exit(1)
        except SidFileError as e:
            sys.stderr.write("ERROR in '%s', %s\n" %
                             (sid_file.input_file_name, e.msg))
            sys.exit(1)
        except FileNotFoundError as e:
            sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
            sys.exit(1)
        except json.decoder.JSONDecodeError as e:
            sys.stderr.write(
                "ERROR in '%s', line %d, column %d, %s\n" %
                (sid_file.input_file_name, e.lineno, e.colno, e.msg))
            sys.exit(1)
Example #24
0
    def post_validate_ctx(self, ctx, modules):
        nbr_option_specified = 0
        if ctx.opts.generate_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.update_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.check_sid_file is not None:
            nbr_option_specified += 1
        if nbr_option_specified == 0:
            return
        if nbr_option_specified > 1:
            sys.stderr.write(
                "Invalid option, only one process on .sid file can be requested.\n"
            )
            return

        fatal_error = False
        for _, etag, _ in ctx.errors:
            if not error.is_warning(error.err_level(etag)):
                fatal_error = True

        if fatal_error or ctx.errors and ctx.opts.check_sid_file is not None:
            sys.stderr.write("Invalid YANG module\n")
            return

        sid_file = SidFile()

        if ctx.opts.sid_registration_info:
            sid_file.sid_registration_info = True

        if ctx.opts.generate_sid_file is not None:
            sid_file.range = ctx.opts.generate_sid_file
            sid_file.is_consistent = False
            sid_file.sid_file_created = True

        if ctx.opts.update_sid_file is not None:
            sid_file.input_file_name = ctx.opts.update_sid_file

        if ctx.opts.check_sid_file is not None:
            sid_file.input_file_name = ctx.opts.check_sid_file
            sid_file.check_consistency = True
            if not sid_file.sid_registration_info:
                print("Checking consistency of '%s'" %
                      sid_file.input_file_name)

        if ctx.opts.extra_sid_range is not None:
            if ctx.opts.update_sid_file is not None:
                sid_file.extra_range = ctx.opts.extra_sid_range
            else:
                sys.stderr.write(
                    "An extra SID range can be specified only during a .sid file update.\n"
                )
                return

        if ctx.opts.list_sid:
            sid_file.list_content = True

        try:
            sid_file.process_sid_file(modules[0])

        except SidParsingError as e:
            sys.stderr.write("ERROR, %s\n" % e)
        except SidFileError as e:
            sys.stderr.write("ERROR in '%s', %s\n" %
                             (sid_file.input_file_name, e))
        except EnvironmentError as e:
            if e.errno == errno.ENOENT:
                sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
            else:
                sys.stderr.write("ERROR, in file '%s' " % e.filename)
        except JSONDecodeError as e:
            sys.stderr.write("ERROR in '%s', %s\n" %
                             (sid_file.input_file_name, e))
        except ValueError as e:
            sys.stderr.write("ERROR in '%s', invalid JSON content\n" %
                             sid_file.input_file_name)
        else:
            sys.exit(0)
        sys.exit(1)
Example #25
0
def check_update(ctx, oldfilename, newmod):
    oldpath = os.pathsep.join(ctx.opts.old_path)
    olddir = os.path.dirname(oldfilename)
    if olddir == '':
        olddir = '.'
    oldpath += os.pathsep + olddir
    oldrepo = pyang.FileRepository(oldpath, use_env=False)
    oldctx = pyang.Context(oldrepo)
    oldctx.opts = ctx.opts
    oldctx.lax_xpath_checks = ctx.lax_xpath_checks
    oldctx.lax_quote_checks = ctx.lax_quote_checks

    if ctx.opts.verbose:
        print("Loading old modules from:")
        for d in oldrepo.dirs:
            print("  %s" % d)
        print("")

    for p in plugin.plugins:
        p.setup_ctx(oldctx)

    for oldfilename in [ctx.opts.check_update_from] + ctx.opts.old_deviation:
        try:
            fd = io.open(oldfilename, "r", encoding="utf-8")
            text = fd.read()
        except IOError as ex:
            sys.stderr.write("error %s: %s\n" % (oldfilename, str(ex)))
            sys.exit(1)
        if oldfilename in ctx.opts.old_deviation:
            oldctx.add_module(oldfilename, text)
        else:
            oldmod = oldctx.add_module(oldfilename, text)
    ctx.errors.extend(oldctx.errors)

    if oldmod is None:
        return

    for (epos, etag, eargs) in ctx.errors:
        if (epos.ref in (newmod.pos.ref, oldmod.pos.ref) and
            error.is_error(error.err_level(etag))):
            return

    if ctx.opts.verbose:
        print("Loaded old modules:")
        for x in oldrepo.get_modules_and_revisions(oldctx):
            (m, r, (fmt, filename)) = x
            print("  %s" % filename)
        print("")

    chk_modulename(oldmod, newmod, ctx)

    chk_namespace(oldmod, newmod, ctx)

    chk_revision(oldmod, newmod, ctx)

    for olds in oldmod.search('feature'):
        chk_feature(olds, newmod, ctx)

    for olds in oldmod.search('identity'):
        chk_identity(olds, newmod, ctx)

    for olds in oldmod.search('typedef'):
        chk_typedef(olds, newmod, ctx)

    for olds in oldmod.search('grouping'):
        chk_grouping(olds, newmod, ctx)

    for olds in oldmod.search('rpc'):
        chk_rpc(olds, newmod, ctx)

    for olds in oldmod.search('notification'):
        chk_notification(olds, newmod, ctx)

    for olds in oldmod.search('extension'):
        chk_extension(olds, newmod, ctx)

    for olds in oldmod.search('augment'):
        chk_augment(olds, newmod, ctx)

    chk_i_children(oldmod, newmod, ctx)
Example #26
0
    context.errors.sort(key=keyfun)

# Parse out all the module names for error checking.
module_names = []
for module in modules:
    module_names.append(module.arg)
    for included_module in module.search('include'):
        module_names.append(included_module.arg)

# Now actually do error parsing
for (epos, etag, eargs) in context.errors:
    if (context.implicit_errors == False and hasattr(epos.top, 'i_modulename')
            and epos.top.arg not in module_names
            and epos.top.i_modulename not in module_names
            and epos.ref not in model_filenames):
        # this module was added implicitly (by import); skip this error
        # the code includes submodules
        continue
    elevel = error.err_level(etag)
    if error.is_warning(elevel):
        kind = "error"
        exit_code = 1
    logging.error('%s: %s: %s', str(epos), kind, etag)

# Now actually do whatever we want.
parsed_modules = [recurse_children(module, module) for module in modules]
if len(parsed_modules) == 1:
    parsed_modules = parsed_modules[0]
with open('testout.json', 'w') as testout_fd:
    json.dump(parsed_modules, testout_fd)
Example #27
0
def run():
    usage = """%prog [options] [<filename>...]
            Validates the YANG module in <filename> (or stdin), and all its dependencies."""

    plugindirs = []
    # check for --plugindir
    idx = 1
    while '--plugindir' in sys.argv[idx:]:
        idx = idx + sys.argv[idx:].index('--plugindir')
        plugindirs.append(sys.argv[idx + 1])
        idx = idx + 1
    plugin.init(plugindirs)

    fmts = {}
    xforms = {}
    for p in plugin.plugins:
        p.add_output_format(fmts)
        p.add_transform(xforms)

    optlist = [
        # use capitalized versions of std options help and version
        optparse.make_option("-h", "--help1",
                             action="help",
                             help="Show this help message and exit ha2"),
        optparse.make_option("-v", "--version",
                             action="version",
                             help="Show version number and exit"),
        optparse.make_option("-V", "--verbose",
                             action="store_true"),
        optparse.make_option("-e", "--list-errors",
                             dest="list_errors",
                             action="store_true",
                             help="Print a listing of all error and warning " \
                                 "codes and exit."),
        optparse.make_option("--print-error-code",
                             dest="print_error_code",
                             action="store_true",
                             help="On errors, print the error code instead " \
                             "of the error message."),
        optparse.make_option("-W",
                             dest="warnings",
                             action="append",
                             default=[],
                             metavar="WARNING",
                             help="If WARNING is 'error', treat all warnings " \
                                 "as errors, except any listed WARNING. " \
                                 "If WARNING is 'none', do not report any " \
                                 "warnings."),
        optparse.make_option("-E",
                             dest="errors",
                             action="append",
                             default=[],
                             metavar="WARNING",
                             help="Treat each WARNING as an error.  For a " \
                                 "list of warnings, use --list-errors."),
        optparse.make_option("--ignore-error",
                             dest="ignore_error_tags",
                             action="append",
                             default=[],
                             metavar="ERROR",
                             help="Ignore ERROR.  Use with care.  For a " \
                                 "list of errors, use --list-errors."),
        optparse.make_option("--ignore-errors",
                             dest="ignore_errors",
                             action="store_true",
                             help="Ignore all errors.  Use with care."),
        optparse.make_option("--canonical",
                             dest="canonical",
                             action="store_true",
                             help="Validate the module(s) according to the " \
                             "canonical YANG order."),
        optparse.make_option("--max-line-length",
                             type="int",
                             dest="max_line_len"),
        optparse.make_option("--max-identifier-length",
                             type="int",
                             dest="max_identifier_len"),
        optparse.make_option("-t", "--transform", dest="transforms",
                             default=[], action="append",
                             help="Apply transform TRANSFORM.  Supported " \
                                  "transforms are: " + ', '.join(list(
                                     xforms.keys()))),
        optparse.make_option("-f", "--format",
                             dest="format",
                             help="Convert to FORMAT.  Supported formats " \
                             "are: " +  ', '.join(list(fmts.keys()))),
        optparse.make_option("-o", "--output",
                             dest="outfile",
                             help="Write the output to OUTFILE instead " \
                             "of stdout."),
        optparse.make_option("-F", "--features",
                             metavar="FEATURES",
                             dest="features",
                             default=[],
                             action="append",
                             help="Features to support, default all. " \
                             "<modname>:[<feature>,]*"),
        optparse.make_option("", "--max-status",
                             metavar="MAXSTATUS",
                             dest="max_status",
                             help="Max status to support, one of: " \
                             "current, deprecated, obsolete"),
        optparse.make_option("", "--deviation-module",
                             metavar="DEVIATION",
                             dest="deviations",
                             default=[],
                             action="append",
                             help="Deviation module"),
        optparse.make_option("-p", "--path",
                             dest="path",
                             default=[],
                             action="append",
                             help=os.pathsep + "-separated search path for yin"
                             " and yang modules"),
        optparse.make_option("--plugindir",
                             dest="plugindir",
                             help="Load pyang plugins from PLUGINDIR"),
        optparse.make_option("--strict",
                             dest="strict",
                             action="store_true",
                             help="Force strict YANG compliance."),
        optparse.make_option("--lax-quote-checks",
                             dest="lax_quote_checks",
                             action="store_true",
                             help="Lax check of backslash in quoted strings."),
        optparse.make_option("--lax-xpath-checks",
                             dest="lax_xpath_checks",
                             action="store_true",
                             help="Lax check of XPath expressions."),
        optparse.make_option("--trim-yin",
                             dest="trim_yin",
                             action="store_true",
                             help="In YIN input modules, trim whitespace "
                             "in textual arguments."),
        optparse.make_option("-L", "--hello",
                             dest="hello",
                             action="store_true",
                             help="Filename of a server's hello message is "
                             "given instead of module filename(s)."),
        optparse.make_option("--keep-comments",
                             dest="keep_comments",
                             action="store_true",
                             help="Pyang will not discard comments; \
                                   has effect if the output plugin can \
                                   handle comments."                                                    ),
        optparse.make_option("--no-path-recurse",
                             dest="no_path_recurse",
                             action="store_true",
                             help="Do not recurse into directories in the \
                                   yang path."                                              ),
        ]

    optparser = optparse.OptionParser(usage, add_help_option=False)
    optparser.version = '%prog ' + pyang.__version__
    optparser.add_options(optlist)

    for p in plugin.plugins:
        p.add_opts(optparser)

    (o, args) = optparser.parse_args()

    if o.list_errors == True:
        for tag in error.error_codes:
            (level, fmt) = error.error_codes[tag]
            if error.is_warning(level):
                print("Warning: %s" % tag)
            elif error.allow_warning(level):
                print("Minor Error:   %s" % tag)
            else:
                print("Error:   %s" % tag)
            print("Message: %s" % fmt)
            print("")
        sys.exit(0)

    if o.outfile != None and o.format == None:
        sys.stderr.write("no format specified\n")
        sys.exit(1)

    # patch the error spec so that -W errors are treated as warnings
    for w in o.warnings:
        if w in error.error_codes:
            (level, wstr) = error.error_codes[w]
            if error.allow_warning(level):
                error.error_codes[w] = (4, wstr)

    filenames = args

    # Parse hello if present
    if o.hello:
        if len(filenames) > 1:
            sys.stderr.write("multiple hello files given\n")
            sys.exit(1)
        if filenames:
            try:
                fd = open(filenames[0], "rb")
            except IOError as ex:
                sys.stderr.write("error %s: %s\n" % (filenames[0], str(ex)))
                sys.exit(1)
        elif sys.version < "3":
            fd = sys.stdin
        else:
            fd = sys.stdin.buffer
        hel = hello.HelloParser().parse(fd)

    path = os.pathsep.join(o.path)

    # add standard search path
    if len(o.path) == 0:
        path = "."
    else:
        path += os.pathsep + "."

    repos = pyang.FileRepository(path,
                                 no_path_recurse=o.no_path_recurse,
                                 verbose=o.verbose)

    ctx = pyang.Context(repos)

    ctx.opts = o
    ctx.canonical = o.canonical
    ctx.max_line_len = o.max_line_len
    ctx.max_identifier_len = o.max_identifier_len
    ctx.trim_yin = o.trim_yin
    ctx.lax_xpath_checks = o.lax_xpath_checks
    ctx.lax_quote_checks = o.lax_quote_checks
    ctx.strict = o.strict
    ctx.max_status = o.max_status

    # make a map of features to support, per module
    if o.hello:
        for (mn, rev) in hel.yang_modules():
            ctx.features[mn] = hel.get_features(mn)
    for f in ctx.opts.features:
        (modulename, features) = parse_features_string(f)
        ctx.features[modulename] = features

    for p in plugin.plugins:
        p.setup_ctx(ctx)

    xform_objs = []
    for transform in o.transforms:
        if transform not in xforms:
            sys.stderr.write("unsupported transform '%s'\n" % transform)
        else:
            xform_obj = xforms[transform]
            xform_obj.setup_xform(ctx)
            xform_objs.append(xform_obj)
    if len(xform_objs) != len(o.transforms):
        sys.exit(1)

    if o.format != None:
        if o.format not in fmts:
            sys.stderr.write("unsupported format '%s'\n" % o.format)
            sys.exit(1)
        emit_obj = fmts[o.format]
        if o.keep_comments and emit_obj.handle_comments:
            ctx.keep_comments = True
        emit_obj.setup_fmt(ctx)
    else:
        emit_obj = None

    xform_and_emit_objs = xform_objs[:]
    if emit_obj is not None:
        xform_and_emit_objs.append(emit_obj)

    for p in plugin.plugins:
        p.pre_load_modules(ctx)

    exit_code = 0
    modules = []

    if o.hello:
        ctx.capabilities = hel.registered_capabilities()
        for (mn, rev) in hel.yang_modules():
            mod = ctx.search_module(0, mn, rev)
            if mod is None:
                emarg = mn
                if rev: emarg += "@" + rev
                sys.stderr.write(
                    "module '%s' specified in hello not found.\n" % emarg)
                sys.exit(1)
            modules.append(mod)
    else:
        if len(filenames) == 0:
            text = sys.stdin.read()
            module = ctx.add_module('<stdin>', text)
            if module is None:
                exit_code = 1
            else:
                modules.append(module)
        if (len(filenames) > 1 and emit_obj is not None
                and not emit_obj.multiple_modules):
            sys.stderr.write("too many files to convert\n")
            sys.exit(1)

        for filename in filenames:
            try:
                fd = io.open(filename, "r", encoding="utf-8")
                text = fd.read()
                if o.verbose:
                    util.report_file_read(filename, "(CL)")
            except IOError as ex:
                sys.stderr.write("error %s: %s\n" % (filename, str(ex)))
                sys.exit(1)
            except UnicodeDecodeError as ex:
                s = str(ex).replace('utf-8', 'utf8')
                sys.stderr.write("%s: unicode error: %s\n" % (filename, s))
                sys.exit(1)
            m = syntax.re_filename.search(filename)
            ctx.yin_module_map = {}
            if m is not None:
                (name, rev, format) = m.groups()
                name = os.path.basename(name)
                module = ctx.add_module(filename,
                                        text,
                                        format,
                                        name,
                                        rev,
                                        expect_failure_error=False)
            else:
                module = ctx.add_module(filename, text)
            if module is None:
                exit_code = 1
            else:
                modules.append(module)

    modulenames = []
    for m in modules:
        modulenames.append(m.arg)
        for s in m.search('include'):
            modulenames.append(s.arg)

    # apply deviations
    for filename in ctx.opts.deviations:
        try:
            fd = io.open(filename, "r", encoding="utf-8")
            text = fd.read()
        except IOError as ex:
            sys.stderr.write("error %s: %s\n" % (filename, str(ex)))
            sys.exit(1)
        except UnicodeDecodeError as ex:
            s = str(ex).replace('utf-8', 'utf8')
            sys.stderr.write("%s: unicode error: %s\n" % (filename, s))
            sys.exit(1)
        m = ctx.add_module(filename, text)
        if m is not None:
            ctx.deviation_modules.append(m)

    for p in plugin.plugins:
        p.pre_validate_ctx(ctx, modules)

    if len(xform_and_emit_objs) > 0 and len(modules) > 0:
        for obj in xform_and_emit_objs:
            obj.pre_validate(ctx, modules)

    ctx.validate()

    for m in modules:
        m.prune()

    # transform modules
    if len(xform_objs) > 0 and len(modules) > 0:
        for xform_obj in xform_objs:
            try:
                if not xform_obj.transform(ctx, modules):
                    for module in modules:
                        module.i_is_validated = False
                        statements.validate_module(ctx, module)
            except error.TransformError as e:
                if e.msg != "":
                    sys.stderr.write(e.msg + '\n')
                sys.exit(e.exit_code)

    # verify the given features
    for m in modules:
        if m.arg in ctx.features:
            for f in ctx.features[m.arg]:
                if f not in m.i_features:
                    sys.stderr.write("unknown feature %s in module %s\n" %
                                     (f, m.arg))
                    sys.exit(1)

    if len(xform_and_emit_objs) > 0 and len(modules) > 0:
        for obj in xform_and_emit_objs:
            obj.post_validate(ctx, modules)

    for p in plugin.plugins:
        p.post_validate_ctx(ctx, modules)

    def keyfun(e):
        if e[0].ref == filenames[0]:
            return 0
        else:
            return 1

    ctx.errors.sort(key=lambda e: (e[0].ref, e[0].line))
    if len(filenames) > 0:
        # first print error for the first filename given
        ctx.errors.sort(key=keyfun)

    if o.ignore_errors:
        ctx.errors = []

    for (epos, etag, eargs) in ctx.errors:
        if etag in o.ignore_error_tags:
            continue
        if (ctx.implicit_errors == False and hasattr(epos.top, 'i_modulename')
                and epos.top.arg not in modulenames
                and epos.top.i_modulename not in modulenames
                and epos.ref not in filenames):
            # this module was added implicitly (by import); skip this error
            # the code includes submodules
            continue
        elevel = error.err_level(etag)
        if error.is_warning(elevel) and etag not in o.errors:
            kind = "warning"
            if 'error' in o.warnings and etag not in o.warnings:
                kind = "error"
                exit_code = 1
            elif 'none' in o.warnings:
                continue
        else:
            kind = "error"
            exit_code = 1
        if o.print_error_code == True:
            sys.stderr.write(str(epos) + ': %s: %s\n' % (kind, etag))
        else:
            sys.stderr.write(str(epos) + ': %s: ' % kind + \
                                 error.err_to_str(etag, eargs) + '\n')

    if emit_obj is not None and len(modules) > 0:
        tmpfile = None
        if o.outfile == None:
            if sys.version < '3':
                fd = codecs.getwriter('utf8')(sys.stdout)
            else:
                fd = sys.stdout
        else:
            tmpfile = o.outfile + ".tmp"
            if sys.version < '3':
                fd = codecs.open(tmpfile, "w+", encoding="utf-8")
            else:
                fd = io.open(tmpfile, "w+", encoding="utf-8")
        try:
            # 调用具体模块的实现
            emit_obj.emit(ctx, modules, fd)
        except error.EmitError as e:
            if e.msg != "":
                sys.stderr.write(e.msg + '\n')
            if tmpfile != None:
                fd.close()
                os.remove(tmpfile)
            sys.exit(e.exit_code)
        except:
            if tmpfile != None:
                fd.close()
                os.remove(tmpfile)
            raise
        if tmpfile != None:
            fd.close()
            os.rename(tmpfile, o.outfile)

    sys.exit(exit_code)
Example #28
0
    def post_validate_ctx(self, ctx, modules):
        nbr_option_specified = 0
        if ctx.opts.generate_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.update_sid_file is not None:
            nbr_option_specified += 1
        if ctx.opts.check_sid_file is not None:
            nbr_option_specified += 1
        if nbr_option_specified == 0:
            return
        if nbr_option_specified > 1:
            sys.stderr.write("Invalid option, only one process on .sid file can be requested.\n")
            return

        fatal_error = False
        for (epos, etag, eargs) in ctx.errors:
            if not error.is_warning(error.err_level(etag)):
                fatal_error = True

        if fatal_error or (ctx.errors != [] and ctx.opts.check_sid_file is not None):
            sys.stderr.write("Invalid YANG module\n")
            return

        sid_file = SidFile()

        if ctx.opts.sid_registration_info:
            sid_file.sid_registration_info = True

        if ctx.opts.generate_sid_file is not None:
            sid_file.range = ctx.opts.generate_sid_file
            sid_file.is_consistent = False
            sid_file.sid_file_created = True

        if ctx.opts.update_sid_file is not None:
            sid_file.input_file_name = ctx.opts.update_sid_file

        if ctx.opts.check_sid_file is not None:
            sid_file.input_file_name = ctx.opts.check_sid_file
            sid_file.check_consistency = True
            if not sid_file.sid_registration_info:
                print("Checking consistency of '%s'" % sid_file.input_file_name)

        if ctx.opts.extra_sid_range is not None:
            if ctx.opts.update_sid_file is not None:
                sid_file.extra_range = ctx.opts.extra_sid_range
            else:
                sys.stderr.write("An extra SID range can be specified only during a .sid file update.\n")
                return

        if ctx.opts.list_sid:
            sid_file.list_content = True

        try:
            sid_file.process_sid_file(modules[0])

        except SidParsingError as e:
            sys.stderr.write("ERROR, %s\n" % e.msg)
            sys.exit(1)
        except SidFileError as e:
            sys.stderr.write("ERROR in '%s', %s\n" % (sid_file.input_file_name, e.msg))
            sys.exit(1)
        except EnvironmentError as e:
            if e.errno == errno.ENOENT:
                sys.stderr.write("ERROR, file '%s' not found\n" % e.filename)
            else:
                sys.stderr.write("ERROR, in file '%s' " % e.filename)
            sys.exit(1)
        except ValueError as e:
            if hasattr(e, "lineno") and sid_file.input_file_name != "*****@*****.**":
                # Present only in python 3.5 and later, except json.decoder.JSONDecodeError
                sys.stderr.write("ERROR in '%s', line %d, column %d, %s\n" % (sid_file.input_file_name, e.lineno, e.colno, e.msg))
            else:
                sys.stderr.write("ERROR in '%s', invalid JSON content\n" % sid_file.input_file_name)
            sys.exit(1)