def expand_from_stream(self, token, stream, previous=None):
"""
Expand macro consuming arguments from the stream
returns the expanded tokens
"""
if self.num_args == 0:
values = []
else:
try:
values = self._parse_macro_actuals(token, stream)
except EOFException:
raise LocationException.warning("EOF reached when parsing `define actuals",
location=token.location)
# Bind defaults
if len(values) < len(self.args):
for i in range(len(values), len(self.args)):
name = self.args[i]
if name in self.defaults:
values.append(self.defaults[name])
else:
raise LocationException.warning(
"Missing value for argument %s" % name,
token.location)
elif len(values) > len(self.args):
raise LocationException.warning("Too many arguments got %i expected %i" %
(len(values), len(self.args)),
token.location)
return self.expand(values, previous)
def expand_from_stream(self, token, stream, previous=None):
"""
Expand macro consuming arguments from the stream
returns the expanded tokens
"""
if self.num_args == 0:
values = []
else:
try:
values = self._parse_macro_actuals(token, stream)
except EOFException:
raise LocationException.warning(
"EOF reached when parsing `define actuals",
location=token.location)
# Bind defaults
if len(values) < len(self.args):
for i in range(len(values), len(self.args)):
name = self.args[i]
if name in self.defaults:
values.append(self.defaults[name])
else:
raise LocationException.warning(
"Missing value for argument %s" % name,
token.location)
elif len(values) > len(self.args):
raise LocationException.warning(
"Too many arguments got %i expected %i" %
(len(values), len(self.args)),
token.location,
)
return self.expand(values, previous)
def find_imports(tokens):
"""
Find imports
"""
results = []
stream = TokenStream(tokens)
while not stream.eof:
token = stream.pop()
if token.kind != IMPORT:
continue
import_token = token
try:
token = stream.pop()
if token.kind == IDENTIFIER:
results.append(token.value)
else:
LocationException.warning(
"import bad argument", token.location
).log(LOGGER)
except EOFException:
LocationException.warning(
"EOF reached when parsing import", location=import_token.location
).log(LOGGER)
return results
def preprocessor(self, # pylint: disable=too-many-arguments,too-many-branches
token, stream, defines, include_paths, included_files):
"""
Handle preprocessor token
"""
if token.value == "define":
macro = define(token, stream)
if macro is not None:
defines[macro.name] = macro
elif token.value == "undef":
undef(token, stream, defines)
elif token.value in ("undefineall", "resetall"):
defines.clear()
elif token.value == "include":
return self.include(token, stream, include_paths, included_files, defines)
elif token.value in ("ifdef", "ifndef"):
try:
tokens = self.if_statement(token, stream, defines)
return self._preprocess(tokens,
defines=defines,
include_paths=include_paths,
included_files=included_files)
except EOFException:
raise LocationException.warning(
"EOF reached when parsing `%s" % token.value,
token.location)
elif token.value in ("celldefine", "endcelldefine", "nounconnected_drive"):
# Ignored
pass
elif token.value in ("timescale", "default_nettype", "unconnected_drive"):
# Ignore directive and arguments
stream.skip_until(NEWLINE)
elif token.value == "pragma":
stream.skip_while(WHITESPACE)
pp_token = stream.pop()
if pp_token.value == "protect":
stream.skip_while(WHITESPACE)
token = stream.pop()
if token.value == "begin_protected":
self._skip_protected_region(stream)
elif token.value in defines:
return self.expand_macro(token, stream, defines, include_paths, included_files)
else:
raise LocationException.debug(
"Verilog undefined name",
token.location)
return []
def preprocessor(self, # pylint: disable=too-many-arguments,too-many-branches
token, stream, defines, include_paths, included_files):
"""
Handle preprocessor token
"""
if token.value == "define":
macro = define(token, stream)
if macro is not None:
defines[macro.name] = macro
elif token.value == "undef":
undef(token, stream, defines)
elif token.value in ("undefineall", "resetall"):
defines.clear()
elif token.value == "include":
return self.include(token, stream, include_paths, included_files, defines)
elif token.value in ("ifdef", "ifndef"):
try:
tokens = self.if_statement(token, stream, defines)
return self._preprocess(tokens,
defines=defines,
include_paths=include_paths,
included_files=included_files)
except EOFException:
raise LocationException.warning(
"EOF reached when parsing `%s" % token.value,
token.location)
elif token.value in ("celldefine", "endcelldefine", "nounconnected_drive"):
# Ignored
pass
elif token.value in ("timescale", "default_nettype", "unconnected_drive"):
# Ignore directive and arguments
stream.skip_until(NEWLINE)
elif token.value == "pragma":
stream.skip_while(WHITESPACE)
pp_token = stream.pop()
if pp_token.value == "protect":
stream.skip_while(WHITESPACE)
token = stream.pop()
if token.value == "begin_protected":
self._skip_protected_region(stream)
elif token.value in defines:
return self.expand_macro(token, stream, defines, include_paths, included_files)
else:
raise LocationException.debug(
"Verilog undefined name",
token.location)
return []
def define(define_token, stream):
"""
Handle a `define directive
"""
stream.skip_while(WHITESPACE, NEWLINE)
try:
name_token = stream.pop()
except EOFException:
raise LocationException.warning("Verilog `define without argument", define_token.location)
if name_token.kind != IDENTIFIER:
raise LocationException.warning("Verilog `define invalid name", name_token.location)
name = name_token.value
try:
token = stream.pop()
except EOFException:
# Empty define
return Macro(name)
if token.kind in (NEWLINE,):
# Empty define
return Macro(name)
if token.kind in (WHITESPACE,):
# Define without arguments
args = tuple()
defaults = {}
elif token.kind == LPAR:
lpar_token = token
args = tuple()
defaults = {}
try:
while token.kind != RPAR:
if token.kind == IDENTIFIER:
argname = token.value
args = args + (argname,)
token = stream.pop()
if token.kind == EQUAL:
token = stream.pop()
defaults[argname] = [token]
token = stream.pop()
else:
token = stream.pop()
except EOFException:
raise LocationException.warning("EOF reached when parsing `define argument list", lpar_token.location)
stream.skip_while(WHITESPACE)
start = stream.idx
end = stream.skip_until(NEWLINE)
if not stream.eof:
stream.pop()
return Macro(name, tokens=stream.slice(start, end), args=args, defaults=defaults)
def check_arg(if_token, arg):
"""
Check the define argument of an if statement
"""
if arg.kind != IDENTIFIER:
raise LocationException.warning("Bad argument to `%s" % if_token.value, arg.location)
stream.skip_while(NEWLINE)
def expand_macro( # pylint: disable=too-many-arguments
self, macro_token, stream, defines, include_paths, included_files):
"""
Expand a macro
"""
macro = defines[macro_token.value]
macro_point = (
strip_previous(macro_token.location),
hash(frozenset(defines.keys())),
)
if macro_point in self._macro_trace:
raise LocationException.error(
"Circular macro expansion of %s detected" % macro_token.value,
macro_token.location,
)
self._macro_trace.add(macro_point)
tokens = self._preprocess(
macro.expand_from_stream(macro_token,
stream,
previous=macro_token.location),
defines=defines,
include_paths=include_paths,
included_files=included_files,
)
self._macro_trace.remove(macro_point)
return tokens
def check_arg(if_token, arg):
"""
Check the define argument of an if statement
"""
if arg.kind != IDENTIFIER:
raise LocationException.warning(
"Bad argument to `%s" % if_token.value, arg.location)
stream.skip_while(NEWLINE)
def undef(undef_token, stream, defines):
"""
Handles undef directive
"""
stream.skip_while(WHITESPACE, NEWLINE)
try:
name_token = stream.pop()
except EOFException:
raise LocationException.warning("EOF reached when parsing `undef", undef_token.location)
if name_token.kind != IDENTIFIER:
raise LocationException.warning("Bad argument to `undef", name_token.location)
if name_token.value not in defines:
raise LocationException.warning("`undef argument was not previously defined", name_token.location)
del defines[name_token.value]
def undef(undef_token, stream, defines):
"""
Handles undef directive
"""
stream.skip_while(WHITESPACE, NEWLINE)
try:
name_token = stream.pop()
except EOFException:
raise LocationException.warning("EOF reached when parsing `undef",
undef_token.location)
if name_token.kind != IDENTIFIER:
raise LocationException.warning("Bad argument to `undef",
name_token.location)
if name_token.value not in defines:
raise LocationException.warning(
"`undef argument was not previously defined", name_token.location)
del defines[name_token.value]
def include(self, token, stream, include_paths, included_files, defines): # pylint: disable=too-many-arguments
"""
Handle `include directive
"""
stream.skip_while(WHITESPACE)
try:
tok = stream.pop()
except EOFException:
raise LocationException.warning("EOF reached when parsing `include argument", token.location)
if tok.kind == PREPROCESSOR:
if tok.value in defines:
macro = defines[tok.value]
else:
raise LocationException.warning("Verilog `include argument not defined", tok.location)
expanded_tokens = self.expand_macro(tok, stream, defines, include_paths, included_files)
if len(expanded_tokens) == 0:
raise LocationException.warning(
"Verilog `include has bad argument, empty define `%s" % macro.name, tok.location
)
if expanded_tokens[0].kind != STRING:
raise LocationException.warning("Verilog `include has bad argument", expanded_tokens[0].location)
file_name_tok = expanded_tokens[0]
elif tok.kind == STRING:
file_name_tok = tok
else:
raise LocationException.warning("Verilog `include bad argument", tok.location)
included_file = find_included_file(include_paths, file_name_tok.value)
included_files.append((file_name_tok.value, included_file))
if included_file is None:
# Is debug message since there are so many builtin includes in tools
raise LocationException.debug(
"Could not find `include file %s" % file_name_tok.value, file_name_tok.location
)
include_point = (strip_previous(token.location), hash(frozenset(defines.keys())))
if include_point in self._include_trace:
raise LocationException.error(
"Circular `include of %s detected" % file_name_tok.value, file_name_tok.location
)
self._include_trace.add(include_point)
included_tokens = self._tokenizer.tokenize(
read_file(included_file), file_name=included_file, previous_location=token.location
)
included_tokens = self._preprocess(included_tokens, defines, include_paths, included_files)
self._include_trace.remove(include_point)
return included_tokens
def _parse_macro_actuals(define_token, stream):
"""
Parse the actual values of macro call such as
1 2 in `macro(1, 2)
"""
stream.skip_while(WHITESPACE)
token = stream.pop()
if token.kind != LPAR:
raise LocationException.warning(
"Bad `define argument list", define_token.location
)
token = stream.pop()
value = []
values = []
bracket_count = 0
brace_count = 0
par_count = 0
while not (token.kind == RPAR and par_count == 0):
if token.kind is LBRACKET:
bracket_count += 1
elif token.kind is RBRACKET:
bracket_count += -1
elif token.kind is LBRACE:
brace_count += 1
elif token.kind is RBRACE:
brace_count += -1
elif token.kind is LPAR:
par_count += 1
elif token.kind is RPAR:
par_count += -1
value_ok = (
token.kind == COMMA
and bracket_count == 0
and brace_count == 0
and par_count == 0
)
if value_ok:
values.append(value)
value = []
else:
value.append(token)
token = stream.pop()
values.append(value)
return values
def find_imports(tokens):
"""
Find imports
"""
results = []
stream = TokenStream(tokens)
while not stream.eof:
token = stream.pop()
if token.kind != IMPORT:
continue
import_token = token
try:
token = stream.pop()
if token.kind == IDENTIFIER:
results.append(token.value)
else:
LocationException.warning("import bad argument",
token.location).log(LOGGER)
except EOFException:
LocationException.warning("EOF reached when parsing import",
location=import_token.location).log(LOGGER)
return results
def _parse_macro_actuals(define_token, stream):
"""
Parse the actual values of macro call such as
1 2 in `macro(1, 2)
"""
stream.skip_while(WHITESPACE)
token = stream.pop()
if token.kind != LPAR:
raise LocationException.warning("Bad `define argument list",
define_token.location)
token = stream.pop()
value = []
values = []
bracket_count = 0
brace_count = 0
par_count = 0
while not (token.kind == RPAR and par_count == 0):
if token.kind is LBRACKET:
bracket_count += 1
elif token.kind is RBRACKET:
bracket_count += -1
elif token.kind is LBRACE:
brace_count += 1
elif token.kind is RBRACE:
brace_count += -1
elif token.kind is LPAR:
par_count += 1
elif token.kind is RPAR:
par_count += -1
value_ok = (token.kind == COMMA
and bracket_count == 0
and brace_count == 0
and par_count == 0)
if value_ok:
values.append(value)
value = []
else:
value.append(token)
token = stream.pop()
values.append(value)
return values
def expand_macro(self, # pylint: disable=too-many-arguments
macro_token, stream, defines, include_paths, included_files):
"""
Expand a macro
"""
macro = defines[macro_token.value]
macro_point = (strip_previous(macro_token.location), hash(frozenset(defines.keys())))
if macro_point in self._macro_trace:
raise LocationException.error(
"Circular macro expansion of %s detected" % macro_token.value,
macro_token.location)
self._macro_trace.add(macro_point)
tokens = self._preprocess(macro.expand_from_stream(macro_token,
stream,
previous=macro_token.location),
defines=defines,
include_paths=include_paths,
included_files=included_files)
self._macro_trace.remove(macro_point)
return tokens
def _parse_macro_actuals(define_token, stream):
"""
Parse the actual values of macro call such as
1 2 in `macro(1, 2)
"""
token = stream.pop()
if token.kind != LPAR:
raise LocationException.warning("Bad `define argument list", define_token.location)
token = stream.pop()
value = []
values = []
while token.kind != RPAR:
if token.kind == COMMA:
values.append(value)
value = []
else:
value.append(token)
token = stream.pop()
values.append(value)
return values
def _parse_macro_actuals(define_token, stream):
"""
Parse the actual values of macro call such as
1 2 in `macro(1, 2)
"""
token = stream.pop()
if token.kind != LPAR:
raise LocationException.warning("Bad `define argument list",
define_token.location)
token = stream.pop()
value = []
values = []
while token.kind != RPAR:
if token.kind == COMMA:
values.append(value)
value = []
else:
value.append(token)
token = stream.pop()
values.append(value)
return values
def define(define_token, stream):
"""
Handle a `define directive
"""
stream.skip_while(WHITESPACE, NEWLINE)
try:
name_token = stream.pop()
except EOFException:
raise LocationException.warning("Verilog `define without argument",
define_token.location)
if name_token.kind != IDENTIFIER:
raise LocationException.warning("Verilog `define invalid name",
name_token.location)
name = name_token.value
try:
token = stream.pop()
except EOFException:
# Empty define
return Macro(name)
if token.kind in (NEWLINE, ):
# Empty define
return Macro(name)
if token.kind in (WHITESPACE, ):
# Define without arguments
args = tuple()
defaults = {}
elif token.kind == LPAR:
lpar_token = token
args = tuple()
defaults = {}
try:
while token.kind != RPAR:
if token.kind == IDENTIFIER:
argname = token.value
args = args + (argname, )
token = stream.pop()
if token.kind == EQUAL:
token = stream.pop()
defaults[argname] = [token]
token = stream.pop()
else:
token = stream.pop()
except EOFException:
raise LocationException.warning(
"EOF reached when parsing `define argument list",
lpar_token.location)
stream.skip_while(WHITESPACE)
start = stream.idx
end = stream.skip_until(NEWLINE)
if not stream.eof:
stream.pop()
return Macro(name,
tokens=stream.slice(start, end),
args=args,
defaults=defaults)
def include( # pylint: disable=too-many-arguments
self, token, stream, include_paths, included_files, defines):
"""
Handle `include directive
"""
stream.skip_while(WHITESPACE)
try:
tok = stream.pop()
except EOFException:
raise LocationException.warning(
"EOF reached when parsing `include argument", token.location)
if tok.kind == PREPROCESSOR:
if tok.value in defines:
macro = defines[tok.value]
else:
raise LocationException.warning(
"Verilog `include argument not defined", tok.location)
expanded_tokens = self.expand_macro(tok, stream, defines,
include_paths, included_files)
# pylint crashes when trying to fix the warning below
if len(expanded_tokens) == 0: # pylint: disable=len-as-condition
raise LocationException.warning(
"Verilog `include has bad argument, empty define `%s" %
macro.name,
tok.location,
)
if expanded_tokens[0].kind != STRING:
raise LocationException.warning(
"Verilog `include has bad argument",
expanded_tokens[0].location)
file_name_tok = expanded_tokens[0]
elif tok.kind == STRING:
file_name_tok = tok
else:
raise LocationException.warning("Verilog `include bad argument",
tok.location)
included_file = find_included_file(include_paths, file_name_tok.value)
included_files.append((file_name_tok.value, included_file))
if included_file is None:
# Is debug message since there are so many builtin includes in tools
raise LocationException.debug(
"Could not find `include file %s" % file_name_tok.value,
file_name_tok.location,
)
include_point = (
strip_previous(token.location),
hash(frozenset(defines.keys())),
)
if include_point in self._include_trace:
raise LocationException.error(
"Circular `include of %s detected" % file_name_tok.value,
file_name_tok.location,
)
self._include_trace.add(include_point)
included_tokens = self._tokenizer.tokenize(
read_file(included_file),
file_name=included_file,
previous_location=token.location,
)
included_tokens = self._preprocess(included_tokens, defines,
include_paths, included_files)
self._include_trace.remove(include_point)
return included_tokens