Beispiel #1
0
    def expand_from_stream(self, token, stream, previous=None):
        """
        Expand macro consuming arguments from the stream
        returns the expanded tokens
        """
        if self.num_args == 0:
            values = []
        else:
            try:
                values = self._parse_macro_actuals(token, stream)
            except EOFException:
                raise LocationException.warning("EOF reached when parsing `define actuals",
                                                location=token.location)

            # Bind defaults
            if len(values) < len(self.args):
                for i in range(len(values), len(self.args)):
                    name = self.args[i]
                    if name in self.defaults:
                        values.append(self.defaults[name])
                    else:
                        raise LocationException.warning(
                            "Missing value for argument %s" % name,
                            token.location)

            elif len(values) > len(self.args):
                raise LocationException.warning("Too many arguments got %i expected %i" %
                                                (len(values), len(self.args)),
                                                token.location)

        return self.expand(values, previous)
Beispiel #2
0
    def expand_from_stream(self, token, stream, previous=None):
        """
        Expand macro consuming arguments from the stream
        returns the expanded tokens
        """
        if self.num_args == 0:
            values = []
        else:
            try:
                values = self._parse_macro_actuals(token, stream)
            except EOFException:
                raise LocationException.warning(
                    "EOF reached when parsing `define actuals",
                    location=token.location)

            # Bind defaults
            if len(values) < len(self.args):
                for i in range(len(values), len(self.args)):
                    name = self.args[i]
                    if name in self.defaults:
                        values.append(self.defaults[name])
                    else:
                        raise LocationException.warning(
                            "Missing value for argument %s" % name,
                            token.location)

            elif len(values) > len(self.args):
                raise LocationException.warning(
                    "Too many arguments got %i expected %i" %
                    (len(values), len(self.args)),
                    token.location,
                )

        return self.expand(values, previous)
Beispiel #3
0
    def find_imports(tokens):
        """
        Find imports
        """
        results = []
        stream = TokenStream(tokens)
        while not stream.eof:
            token = stream.pop()

            if token.kind != IMPORT:
                continue
            import_token = token
            try:
                token = stream.pop()
                if token.kind == IDENTIFIER:
                    results.append(token.value)
                else:
                    LocationException.warning(
                        "import bad argument", token.location
                    ).log(LOGGER)
            except EOFException:
                LocationException.warning(
                    "EOF reached when parsing import", location=import_token.location
                ).log(LOGGER)
        return results
Beispiel #4
0
    def preprocessor(self,  # pylint: disable=too-many-arguments,too-many-branches
                     token, stream, defines, include_paths, included_files):
        """
        Handle preprocessor token
        """
        if token.value == "define":
            macro = define(token, stream)
            if macro is not None:
                defines[macro.name] = macro

        elif token.value == "undef":
            undef(token, stream, defines)

        elif token.value in ("undefineall", "resetall"):
            defines.clear()

        elif token.value == "include":
            return self.include(token, stream, include_paths, included_files, defines)

        elif token.value in ("ifdef", "ifndef"):
            try:
                tokens = self.if_statement(token, stream, defines)
                return self._preprocess(tokens,
                                        defines=defines,
                                        include_paths=include_paths,
                                        included_files=included_files)
            except EOFException:
                raise LocationException.warning(
                    "EOF reached when parsing `%s" % token.value,
                    token.location)

        elif token.value in ("celldefine", "endcelldefine", "nounconnected_drive"):
            # Ignored
            pass

        elif token.value in ("timescale", "default_nettype", "unconnected_drive"):
            # Ignore directive and arguments
            stream.skip_until(NEWLINE)

        elif token.value == "pragma":
            stream.skip_while(WHITESPACE)
            pp_token = stream.pop()

            if pp_token.value == "protect":
                stream.skip_while(WHITESPACE)
                token = stream.pop()

                if token.value == "begin_protected":
                    self._skip_protected_region(stream)

        elif token.value in defines:
            return self.expand_macro(token, stream, defines, include_paths, included_files)
        else:
            raise LocationException.debug(
                "Verilog undefined name",
                token.location)

        return []
Beispiel #5
0
    def preprocessor(self,  # pylint: disable=too-many-arguments,too-many-branches
                     token, stream, defines, include_paths, included_files):
        """
        Handle preprocessor token
        """
        if token.value == "define":
            macro = define(token, stream)
            if macro is not None:
                defines[macro.name] = macro

        elif token.value == "undef":
            undef(token, stream, defines)

        elif token.value in ("undefineall", "resetall"):
            defines.clear()

        elif token.value == "include":
            return self.include(token, stream, include_paths, included_files, defines)

        elif token.value in ("ifdef", "ifndef"):
            try:
                tokens = self.if_statement(token, stream, defines)
                return self._preprocess(tokens,
                                        defines=defines,
                                        include_paths=include_paths,
                                        included_files=included_files)
            except EOFException:
                raise LocationException.warning(
                    "EOF reached when parsing `%s" % token.value,
                    token.location)

        elif token.value in ("celldefine", "endcelldefine", "nounconnected_drive"):
            # Ignored
            pass

        elif token.value in ("timescale", "default_nettype", "unconnected_drive"):
            # Ignore directive and arguments
            stream.skip_until(NEWLINE)

        elif token.value == "pragma":
            stream.skip_while(WHITESPACE)
            pp_token = stream.pop()

            if pp_token.value == "protect":
                stream.skip_while(WHITESPACE)
                token = stream.pop()

                if token.value == "begin_protected":
                    self._skip_protected_region(stream)

        elif token.value in defines:
            return self.expand_macro(token, stream, defines, include_paths, included_files)
        else:
            raise LocationException.debug(
                "Verilog undefined name",
                token.location)

        return []
Beispiel #6
0
def define(define_token, stream):
    """
    Handle a `define directive
    """
    stream.skip_while(WHITESPACE, NEWLINE)
    try:
        name_token = stream.pop()
    except EOFException:
        raise LocationException.warning("Verilog `define without argument", define_token.location)

    if name_token.kind != IDENTIFIER:
        raise LocationException.warning("Verilog `define invalid name", name_token.location)

    name = name_token.value

    try:
        token = stream.pop()
    except EOFException:
        # Empty define
        return Macro(name)

    if token.kind in (NEWLINE,):
        # Empty define
        return Macro(name)

    if token.kind in (WHITESPACE,):
        # Define without arguments
        args = tuple()
        defaults = {}
    elif token.kind == LPAR:
        lpar_token = token
        args = tuple()
        defaults = {}

        try:
            while token.kind != RPAR:
                if token.kind == IDENTIFIER:
                    argname = token.value
                    args = args + (argname,)
                    token = stream.pop()
                    if token.kind == EQUAL:
                        token = stream.pop()
                        defaults[argname] = [token]
                        token = stream.pop()
                else:
                    token = stream.pop()
        except EOFException:
            raise LocationException.warning("EOF reached when parsing `define argument list", lpar_token.location)

    stream.skip_while(WHITESPACE)
    start = stream.idx
    end = stream.skip_until(NEWLINE)
    if not stream.eof:
        stream.pop()
    return Macro(name, tokens=stream.slice(start, end), args=args, defaults=defaults)
Beispiel #7
0
 def check_arg(if_token, arg):
     """
     Check the define argument of an if statement
     """
     if arg.kind != IDENTIFIER:
         raise LocationException.warning("Bad argument to `%s" % if_token.value, arg.location)
     stream.skip_while(NEWLINE)
Beispiel #8
0
 def expand_macro(  # pylint: disable=too-many-arguments
         self, macro_token, stream, defines, include_paths, included_files):
     """
     Expand a macro
     """
     macro = defines[macro_token.value]
     macro_point = (
         strip_previous(macro_token.location),
         hash(frozenset(defines.keys())),
     )
     if macro_point in self._macro_trace:
         raise LocationException.error(
             "Circular macro expansion of %s detected" % macro_token.value,
             macro_token.location,
         )
     self._macro_trace.add(macro_point)
     tokens = self._preprocess(
         macro.expand_from_stream(macro_token,
                                  stream,
                                  previous=macro_token.location),
         defines=defines,
         include_paths=include_paths,
         included_files=included_files,
     )
     self._macro_trace.remove(macro_point)
     return tokens
Beispiel #9
0
 def check_arg(if_token, arg):
     """
     Check the define argument of an if statement
     """
     if arg.kind != IDENTIFIER:
         raise LocationException.warning(
             "Bad argument to `%s" % if_token.value, arg.location)
     stream.skip_while(NEWLINE)
Beispiel #10
0
def undef(undef_token, stream, defines):
    """
    Handles undef directive
    """
    stream.skip_while(WHITESPACE, NEWLINE)
    try:
        name_token = stream.pop()
    except EOFException:
        raise LocationException.warning("EOF reached when parsing `undef", undef_token.location)

    if name_token.kind != IDENTIFIER:
        raise LocationException.warning("Bad argument to `undef", name_token.location)

    if name_token.value not in defines:
        raise LocationException.warning("`undef argument was not previously defined", name_token.location)

    del defines[name_token.value]
Beispiel #11
0
def undef(undef_token, stream, defines):
    """
    Handles undef directive
    """
    stream.skip_while(WHITESPACE, NEWLINE)
    try:
        name_token = stream.pop()
    except EOFException:
        raise LocationException.warning("EOF reached when parsing `undef",
                                        undef_token.location)

    if name_token.kind != IDENTIFIER:
        raise LocationException.warning("Bad argument to `undef",
                                        name_token.location)

    if name_token.value not in defines:
        raise LocationException.warning(
            "`undef argument was not previously defined", name_token.location)

    del defines[name_token.value]
Beispiel #12
0
    def include(self, token, stream, include_paths, included_files, defines):  # pylint: disable=too-many-arguments
        """
        Handle `include directive
        """
        stream.skip_while(WHITESPACE)
        try:
            tok = stream.pop()
        except EOFException:
            raise LocationException.warning("EOF reached when parsing `include argument", token.location)

        if tok.kind == PREPROCESSOR:
            if tok.value in defines:
                macro = defines[tok.value]
            else:
                raise LocationException.warning("Verilog `include argument not defined", tok.location)

            expanded_tokens = self.expand_macro(tok, stream, defines, include_paths, included_files)

            if len(expanded_tokens) == 0:
                raise LocationException.warning(
                    "Verilog `include has bad argument, empty define `%s" % macro.name, tok.location
                )

            if expanded_tokens[0].kind != STRING:
                raise LocationException.warning("Verilog `include has bad argument", expanded_tokens[0].location)

            file_name_tok = expanded_tokens[0]

        elif tok.kind == STRING:
            file_name_tok = tok
        else:
            raise LocationException.warning("Verilog `include bad argument", tok.location)

        included_file = find_included_file(include_paths, file_name_tok.value)
        included_files.append((file_name_tok.value, included_file))
        if included_file is None:
            # Is debug message since there are so many builtin includes in tools
            raise LocationException.debug(
                "Could not find `include file %s" % file_name_tok.value, file_name_tok.location
            )

        include_point = (strip_previous(token.location), hash(frozenset(defines.keys())))
        if include_point in self._include_trace:
            raise LocationException.error(
                "Circular `include of %s detected" % file_name_tok.value, file_name_tok.location
            )
        self._include_trace.add(include_point)

        included_tokens = self._tokenizer.tokenize(
            read_file(included_file), file_name=included_file, previous_location=token.location
        )
        included_tokens = self._preprocess(included_tokens, defines, include_paths, included_files)
        self._include_trace.remove(include_point)
        return included_tokens
Beispiel #13
0
    def _parse_macro_actuals(define_token, stream):
        """
        Parse the actual values of macro call such as
        1 2 in `macro(1, 2)
        """

        stream.skip_while(WHITESPACE)

        token = stream.pop()
        if token.kind != LPAR:
            raise LocationException.warning(
                "Bad `define argument list", define_token.location
            )
        token = stream.pop()
        value = []
        values = []

        bracket_count = 0
        brace_count = 0
        par_count = 0

        while not (token.kind == RPAR and par_count == 0):
            if token.kind is LBRACKET:
                bracket_count += 1
            elif token.kind is RBRACKET:
                bracket_count += -1
            elif token.kind is LBRACE:
                brace_count += 1
            elif token.kind is RBRACE:
                brace_count += -1
            elif token.kind is LPAR:
                par_count += 1
            elif token.kind is RPAR:
                par_count += -1

            value_ok = (
                token.kind == COMMA
                and bracket_count == 0
                and brace_count == 0
                and par_count == 0
            )

            if value_ok:
                values.append(value)
                value = []
            else:
                value.append(token)
            token = stream.pop()

        values.append(value)
        return values
Beispiel #14
0
    def find_imports(tokens):
        """
        Find imports
        """
        results = []
        stream = TokenStream(tokens)
        while not stream.eof:
            token = stream.pop()

            if token.kind != IMPORT:
                continue
            import_token = token
            try:
                token = stream.pop()
                if token.kind == IDENTIFIER:
                    results.append(token.value)
                else:
                    LocationException.warning("import bad argument",
                                              token.location).log(LOGGER)
            except EOFException:
                LocationException.warning("EOF reached when parsing import",
                                          location=import_token.location).log(LOGGER)
        return results
Beispiel #15
0
    def _parse_macro_actuals(define_token, stream):
        """
        Parse the actual values of macro call such as
        1 2 in `macro(1, 2)
        """

        stream.skip_while(WHITESPACE)

        token = stream.pop()
        if token.kind != LPAR:
            raise LocationException.warning("Bad `define argument list",
                                            define_token.location)
        token = stream.pop()
        value = []
        values = []

        bracket_count = 0
        brace_count = 0
        par_count = 0

        while not (token.kind == RPAR and par_count == 0):
            if token.kind is LBRACKET:
                bracket_count += 1
            elif token.kind is RBRACKET:
                bracket_count += -1
            elif token.kind is LBRACE:
                brace_count += 1
            elif token.kind is RBRACE:
                brace_count += -1
            elif token.kind is LPAR:
                par_count += 1
            elif token.kind is RPAR:
                par_count += -1

            value_ok = (token.kind == COMMA
                        and bracket_count == 0
                        and brace_count == 0
                        and par_count == 0)

            if value_ok:
                values.append(value)
                value = []
            else:
                value.append(token)
            token = stream.pop()

        values.append(value)
        return values
Beispiel #16
0
 def expand_macro(self,  # pylint: disable=too-many-arguments
                  macro_token, stream, defines, include_paths, included_files):
     """
     Expand a macro
     """
     macro = defines[macro_token.value]
     macro_point = (strip_previous(macro_token.location), hash(frozenset(defines.keys())))
     if macro_point in self._macro_trace:
         raise LocationException.error(
             "Circular macro expansion of %s detected" % macro_token.value,
             macro_token.location)
     self._macro_trace.add(macro_point)
     tokens = self._preprocess(macro.expand_from_stream(macro_token,
                                                        stream,
                                                        previous=macro_token.location),
                               defines=defines,
                               include_paths=include_paths,
                               included_files=included_files)
     self._macro_trace.remove(macro_point)
     return tokens
Beispiel #17
0
    def _parse_macro_actuals(define_token, stream):
        """
        Parse the actual values of macro call such as
        1 2 in `macro(1, 2)
        """
        token = stream.pop()
        if token.kind != LPAR:
            raise LocationException.warning("Bad `define argument list", define_token.location)
        token = stream.pop()
        value = []
        values = []
        while token.kind != RPAR:
            if token.kind == COMMA:
                values.append(value)
                value = []
            else:
                value.append(token)
            token = stream.pop()

        values.append(value)
        return values
Beispiel #18
0
    def _parse_macro_actuals(define_token, stream):
        """
        Parse the actual values of macro call such as
        1 2 in `macro(1, 2)
        """
        token = stream.pop()
        if token.kind != LPAR:
            raise LocationException.warning("Bad `define argument list",
                                            define_token.location)
        token = stream.pop()
        value = []
        values = []
        while token.kind != RPAR:
            if token.kind == COMMA:
                values.append(value)
                value = []
            else:
                value.append(token)
            token = stream.pop()

        values.append(value)
        return values
Beispiel #19
0
def define(define_token, stream):
    """
    Handle a `define directive
    """
    stream.skip_while(WHITESPACE, NEWLINE)
    try:
        name_token = stream.pop()
    except EOFException:
        raise LocationException.warning("Verilog `define without argument",
                                        define_token.location)

    if name_token.kind != IDENTIFIER:
        raise LocationException.warning("Verilog `define invalid name",
                                        name_token.location)

    name = name_token.value

    try:
        token = stream.pop()
    except EOFException:
        # Empty define
        return Macro(name)

    if token.kind in (NEWLINE, ):
        # Empty define
        return Macro(name)

    if token.kind in (WHITESPACE, ):
        # Define without arguments
        args = tuple()
        defaults = {}
    elif token.kind == LPAR:
        lpar_token = token
        args = tuple()
        defaults = {}

        try:
            while token.kind != RPAR:
                if token.kind == IDENTIFIER:
                    argname = token.value
                    args = args + (argname, )
                    token = stream.pop()
                    if token.kind == EQUAL:
                        token = stream.pop()
                        defaults[argname] = [token]
                        token = stream.pop()
                else:
                    token = stream.pop()
        except EOFException:
            raise LocationException.warning(
                "EOF reached when parsing `define argument list",
                lpar_token.location)

    stream.skip_while(WHITESPACE)
    start = stream.idx
    end = stream.skip_until(NEWLINE)
    if not stream.eof:
        stream.pop()
    return Macro(name,
                 tokens=stream.slice(start, end),
                 args=args,
                 defaults=defaults)
Beispiel #20
0
    def include(  # pylint: disable=too-many-arguments
            self, token, stream, include_paths, included_files, defines):
        """
        Handle `include directive
        """
        stream.skip_while(WHITESPACE)
        try:
            tok = stream.pop()
        except EOFException:
            raise LocationException.warning(
                "EOF reached when parsing `include argument", token.location)

        if tok.kind == PREPROCESSOR:
            if tok.value in defines:
                macro = defines[tok.value]
            else:
                raise LocationException.warning(
                    "Verilog `include argument not defined", tok.location)

            expanded_tokens = self.expand_macro(tok, stream, defines,
                                                include_paths, included_files)

            # pylint crashes when trying to fix the warning below
            if len(expanded_tokens) == 0:  # pylint: disable=len-as-condition
                raise LocationException.warning(
                    "Verilog `include has bad argument, empty define `%s" %
                    macro.name,
                    tok.location,
                )

            if expanded_tokens[0].kind != STRING:
                raise LocationException.warning(
                    "Verilog `include has bad argument",
                    expanded_tokens[0].location)

            file_name_tok = expanded_tokens[0]

        elif tok.kind == STRING:
            file_name_tok = tok
        else:
            raise LocationException.warning("Verilog `include bad argument",
                                            tok.location)

        included_file = find_included_file(include_paths, file_name_tok.value)
        included_files.append((file_name_tok.value, included_file))
        if included_file is None:
            # Is debug message since there are so many builtin includes in tools
            raise LocationException.debug(
                "Could not find `include file %s" % file_name_tok.value,
                file_name_tok.location,
            )

        include_point = (
            strip_previous(token.location),
            hash(frozenset(defines.keys())),
        )
        if include_point in self._include_trace:
            raise LocationException.error(
                "Circular `include of %s detected" % file_name_tok.value,
                file_name_tok.location,
            )
        self._include_trace.add(include_point)

        included_tokens = self._tokenizer.tokenize(
            read_file(included_file),
            file_name=included_file,
            previous_location=token.location,
        )
        included_tokens = self._preprocess(included_tokens, defines,
                                           include_paths, included_files)
        self._include_trace.remove(include_point)
        return included_tokens