def expand_from_stream(self, token, stream, previous=None): """ Expand macro consuming arguments from the stream returns the expanded tokens """ if self.num_args == 0: values = [] else: try: values = self._parse_macro_actuals(token, stream) except EOFException: raise LocationException.warning("EOF reached when parsing `define actuals", location=token.location) # Bind defaults if len(values) < len(self.args): for i in range(len(values), len(self.args)): name = self.args[i] if name in self.defaults: values.append(self.defaults[name]) else: raise LocationException.warning( "Missing value for argument %s" % name, token.location) elif len(values) > len(self.args): raise LocationException.warning("Too many arguments got %i expected %i" % (len(values), len(self.args)), token.location) return self.expand(values, previous)
def expand_from_stream(self, token, stream, previous=None): """ Expand macro consuming arguments from the stream returns the expanded tokens """ if self.num_args == 0: values = [] else: try: values = self._parse_macro_actuals(token, stream) except EOFException: raise LocationException.warning( "EOF reached when parsing `define actuals", location=token.location) # Bind defaults if len(values) < len(self.args): for i in range(len(values), len(self.args)): name = self.args[i] if name in self.defaults: values.append(self.defaults[name]) else: raise LocationException.warning( "Missing value for argument %s" % name, token.location) elif len(values) > len(self.args): raise LocationException.warning( "Too many arguments got %i expected %i" % (len(values), len(self.args)), token.location, ) return self.expand(values, previous)
def find_imports(tokens): """ Find imports """ results = [] stream = TokenStream(tokens) while not stream.eof: token = stream.pop() if token.kind != IMPORT: continue import_token = token try: token = stream.pop() if token.kind == IDENTIFIER: results.append(token.value) else: LocationException.warning( "import bad argument", token.location ).log(LOGGER) except EOFException: LocationException.warning( "EOF reached when parsing import", location=import_token.location ).log(LOGGER) return results
def preprocessor(self, # pylint: disable=too-many-arguments,too-many-branches token, stream, defines, include_paths, included_files): """ Handle preprocessor token """ if token.value == "define": macro = define(token, stream) if macro is not None: defines[macro.name] = macro elif token.value == "undef": undef(token, stream, defines) elif token.value in ("undefineall", "resetall"): defines.clear() elif token.value == "include": return self.include(token, stream, include_paths, included_files, defines) elif token.value in ("ifdef", "ifndef"): try: tokens = self.if_statement(token, stream, defines) return self._preprocess(tokens, defines=defines, include_paths=include_paths, included_files=included_files) except EOFException: raise LocationException.warning( "EOF reached when parsing `%s" % token.value, token.location) elif token.value in ("celldefine", "endcelldefine", "nounconnected_drive"): # Ignored pass elif token.value in ("timescale", "default_nettype", "unconnected_drive"): # Ignore directive and arguments stream.skip_until(NEWLINE) elif token.value == "pragma": stream.skip_while(WHITESPACE) pp_token = stream.pop() if pp_token.value == "protect": stream.skip_while(WHITESPACE) token = stream.pop() if token.value == "begin_protected": self._skip_protected_region(stream) elif token.value in defines: return self.expand_macro(token, stream, defines, include_paths, included_files) else: raise LocationException.debug( "Verilog undefined name", token.location) return []
def define(define_token, stream): """ Handle a `define directive """ stream.skip_while(WHITESPACE, NEWLINE) try: name_token = stream.pop() except EOFException: raise LocationException.warning("Verilog `define without argument", define_token.location) if name_token.kind != IDENTIFIER: raise LocationException.warning("Verilog `define invalid name", name_token.location) name = name_token.value try: token = stream.pop() except EOFException: # Empty define return Macro(name) if token.kind in (NEWLINE,): # Empty define return Macro(name) if token.kind in (WHITESPACE,): # Define without arguments args = tuple() defaults = {} elif token.kind == LPAR: lpar_token = token args = tuple() defaults = {} try: while token.kind != RPAR: if token.kind == IDENTIFIER: argname = token.value args = args + (argname,) token = stream.pop() if token.kind == EQUAL: token = stream.pop() defaults[argname] = [token] token = stream.pop() else: token = stream.pop() except EOFException: raise LocationException.warning("EOF reached when parsing `define argument list", lpar_token.location) stream.skip_while(WHITESPACE) start = stream.idx end = stream.skip_until(NEWLINE) if not stream.eof: stream.pop() return Macro(name, tokens=stream.slice(start, end), args=args, defaults=defaults)
def check_arg(if_token, arg): """ Check the define argument of an if statement """ if arg.kind != IDENTIFIER: raise LocationException.warning("Bad argument to `%s" % if_token.value, arg.location) stream.skip_while(NEWLINE)
def expand_macro( # pylint: disable=too-many-arguments self, macro_token, stream, defines, include_paths, included_files): """ Expand a macro """ macro = defines[macro_token.value] macro_point = ( strip_previous(macro_token.location), hash(frozenset(defines.keys())), ) if macro_point in self._macro_trace: raise LocationException.error( "Circular macro expansion of %s detected" % macro_token.value, macro_token.location, ) self._macro_trace.add(macro_point) tokens = self._preprocess( macro.expand_from_stream(macro_token, stream, previous=macro_token.location), defines=defines, include_paths=include_paths, included_files=included_files, ) self._macro_trace.remove(macro_point) return tokens
def check_arg(if_token, arg): """ Check the define argument of an if statement """ if arg.kind != IDENTIFIER: raise LocationException.warning( "Bad argument to `%s" % if_token.value, arg.location) stream.skip_while(NEWLINE)
def undef(undef_token, stream, defines): """ Handles undef directive """ stream.skip_while(WHITESPACE, NEWLINE) try: name_token = stream.pop() except EOFException: raise LocationException.warning("EOF reached when parsing `undef", undef_token.location) if name_token.kind != IDENTIFIER: raise LocationException.warning("Bad argument to `undef", name_token.location) if name_token.value not in defines: raise LocationException.warning("`undef argument was not previously defined", name_token.location) del defines[name_token.value]
def undef(undef_token, stream, defines): """ Handles undef directive """ stream.skip_while(WHITESPACE, NEWLINE) try: name_token = stream.pop() except EOFException: raise LocationException.warning("EOF reached when parsing `undef", undef_token.location) if name_token.kind != IDENTIFIER: raise LocationException.warning("Bad argument to `undef", name_token.location) if name_token.value not in defines: raise LocationException.warning( "`undef argument was not previously defined", name_token.location) del defines[name_token.value]
def include(self, token, stream, include_paths, included_files, defines): # pylint: disable=too-many-arguments """ Handle `include directive """ stream.skip_while(WHITESPACE) try: tok = stream.pop() except EOFException: raise LocationException.warning("EOF reached when parsing `include argument", token.location) if tok.kind == PREPROCESSOR: if tok.value in defines: macro = defines[tok.value] else: raise LocationException.warning("Verilog `include argument not defined", tok.location) expanded_tokens = self.expand_macro(tok, stream, defines, include_paths, included_files) if len(expanded_tokens) == 0: raise LocationException.warning( "Verilog `include has bad argument, empty define `%s" % macro.name, tok.location ) if expanded_tokens[0].kind != STRING: raise LocationException.warning("Verilog `include has bad argument", expanded_tokens[0].location) file_name_tok = expanded_tokens[0] elif tok.kind == STRING: file_name_tok = tok else: raise LocationException.warning("Verilog `include bad argument", tok.location) included_file = find_included_file(include_paths, file_name_tok.value) included_files.append((file_name_tok.value, included_file)) if included_file is None: # Is debug message since there are so many builtin includes in tools raise LocationException.debug( "Could not find `include file %s" % file_name_tok.value, file_name_tok.location ) include_point = (strip_previous(token.location), hash(frozenset(defines.keys()))) if include_point in self._include_trace: raise LocationException.error( "Circular `include of %s detected" % file_name_tok.value, file_name_tok.location ) self._include_trace.add(include_point) included_tokens = self._tokenizer.tokenize( read_file(included_file), file_name=included_file, previous_location=token.location ) included_tokens = self._preprocess(included_tokens, defines, include_paths, included_files) self._include_trace.remove(include_point) return included_tokens
def _parse_macro_actuals(define_token, stream): """ Parse the actual values of macro call such as 1 2 in `macro(1, 2) """ stream.skip_while(WHITESPACE) token = stream.pop() if token.kind != LPAR: raise LocationException.warning( "Bad `define argument list", define_token.location ) token = stream.pop() value = [] values = [] bracket_count = 0 brace_count = 0 par_count = 0 while not (token.kind == RPAR and par_count == 0): if token.kind is LBRACKET: bracket_count += 1 elif token.kind is RBRACKET: bracket_count += -1 elif token.kind is LBRACE: brace_count += 1 elif token.kind is RBRACE: brace_count += -1 elif token.kind is LPAR: par_count += 1 elif token.kind is RPAR: par_count += -1 value_ok = ( token.kind == COMMA and bracket_count == 0 and brace_count == 0 and par_count == 0 ) if value_ok: values.append(value) value = [] else: value.append(token) token = stream.pop() values.append(value) return values
def find_imports(tokens): """ Find imports """ results = [] stream = TokenStream(tokens) while not stream.eof: token = stream.pop() if token.kind != IMPORT: continue import_token = token try: token = stream.pop() if token.kind == IDENTIFIER: results.append(token.value) else: LocationException.warning("import bad argument", token.location).log(LOGGER) except EOFException: LocationException.warning("EOF reached when parsing import", location=import_token.location).log(LOGGER) return results
def _parse_macro_actuals(define_token, stream): """ Parse the actual values of macro call such as 1 2 in `macro(1, 2) """ stream.skip_while(WHITESPACE) token = stream.pop() if token.kind != LPAR: raise LocationException.warning("Bad `define argument list", define_token.location) token = stream.pop() value = [] values = [] bracket_count = 0 brace_count = 0 par_count = 0 while not (token.kind == RPAR and par_count == 0): if token.kind is LBRACKET: bracket_count += 1 elif token.kind is RBRACKET: bracket_count += -1 elif token.kind is LBRACE: brace_count += 1 elif token.kind is RBRACE: brace_count += -1 elif token.kind is LPAR: par_count += 1 elif token.kind is RPAR: par_count += -1 value_ok = (token.kind == COMMA and bracket_count == 0 and brace_count == 0 and par_count == 0) if value_ok: values.append(value) value = [] else: value.append(token) token = stream.pop() values.append(value) return values
def expand_macro(self, # pylint: disable=too-many-arguments macro_token, stream, defines, include_paths, included_files): """ Expand a macro """ macro = defines[macro_token.value] macro_point = (strip_previous(macro_token.location), hash(frozenset(defines.keys()))) if macro_point in self._macro_trace: raise LocationException.error( "Circular macro expansion of %s detected" % macro_token.value, macro_token.location) self._macro_trace.add(macro_point) tokens = self._preprocess(macro.expand_from_stream(macro_token, stream, previous=macro_token.location), defines=defines, include_paths=include_paths, included_files=included_files) self._macro_trace.remove(macro_point) return tokens
def _parse_macro_actuals(define_token, stream): """ Parse the actual values of macro call such as 1 2 in `macro(1, 2) """ token = stream.pop() if token.kind != LPAR: raise LocationException.warning("Bad `define argument list", define_token.location) token = stream.pop() value = [] values = [] while token.kind != RPAR: if token.kind == COMMA: values.append(value) value = [] else: value.append(token) token = stream.pop() values.append(value) return values
def define(define_token, stream): """ Handle a `define directive """ stream.skip_while(WHITESPACE, NEWLINE) try: name_token = stream.pop() except EOFException: raise LocationException.warning("Verilog `define without argument", define_token.location) if name_token.kind != IDENTIFIER: raise LocationException.warning("Verilog `define invalid name", name_token.location) name = name_token.value try: token = stream.pop() except EOFException: # Empty define return Macro(name) if token.kind in (NEWLINE, ): # Empty define return Macro(name) if token.kind in (WHITESPACE, ): # Define without arguments args = tuple() defaults = {} elif token.kind == LPAR: lpar_token = token args = tuple() defaults = {} try: while token.kind != RPAR: if token.kind == IDENTIFIER: argname = token.value args = args + (argname, ) token = stream.pop() if token.kind == EQUAL: token = stream.pop() defaults[argname] = [token] token = stream.pop() else: token = stream.pop() except EOFException: raise LocationException.warning( "EOF reached when parsing `define argument list", lpar_token.location) stream.skip_while(WHITESPACE) start = stream.idx end = stream.skip_until(NEWLINE) if not stream.eof: stream.pop() return Macro(name, tokens=stream.slice(start, end), args=args, defaults=defaults)
def include( # pylint: disable=too-many-arguments self, token, stream, include_paths, included_files, defines): """ Handle `include directive """ stream.skip_while(WHITESPACE) try: tok = stream.pop() except EOFException: raise LocationException.warning( "EOF reached when parsing `include argument", token.location) if tok.kind == PREPROCESSOR: if tok.value in defines: macro = defines[tok.value] else: raise LocationException.warning( "Verilog `include argument not defined", tok.location) expanded_tokens = self.expand_macro(tok, stream, defines, include_paths, included_files) # pylint crashes when trying to fix the warning below if len(expanded_tokens) == 0: # pylint: disable=len-as-condition raise LocationException.warning( "Verilog `include has bad argument, empty define `%s" % macro.name, tok.location, ) if expanded_tokens[0].kind != STRING: raise LocationException.warning( "Verilog `include has bad argument", expanded_tokens[0].location) file_name_tok = expanded_tokens[0] elif tok.kind == STRING: file_name_tok = tok else: raise LocationException.warning("Verilog `include bad argument", tok.location) included_file = find_included_file(include_paths, file_name_tok.value) included_files.append((file_name_tok.value, included_file)) if included_file is None: # Is debug message since there are so many builtin includes in tools raise LocationException.debug( "Could not find `include file %s" % file_name_tok.value, file_name_tok.location, ) include_point = ( strip_previous(token.location), hash(frozenset(defines.keys())), ) if include_point in self._include_trace: raise LocationException.error( "Circular `include of %s detected" % file_name_tok.value, file_name_tok.location, ) self._include_trace.add(include_point) included_tokens = self._tokenizer.tokenize( read_file(included_file), file_name=included_file, previous_location=token.location, ) included_tokens = self._preprocess(included_tokens, defines, include_paths, included_files) self._include_trace.remove(include_point) return included_tokens