def stateLinebreak(cls, parserState: ParserState):
token = parserState.Token
if isinstance(token, SpaceToken):
parserState.NewToken = IndentationToken(token)
parserState.NewBlock = IndentationBlock(parserState.LastBlock, parserState.NewToken)
parserState.Pop()
else:
parserState.Pop()
if (parserState.TokenMarker is None):
# print(" new marker: None -> {0!s}".format(token))
parserState.TokenMarker = token
# print(" {DARK_GREEN}re-issue: {GREEN}{state!s} {DARK_GREEN}token={GREEN}{token}{NOCOLOR}".format(state=parserState, token=parserState.Token, **Console.Foreground))
parserState.NextState(parserState)
def stateDeclarativeRegion(cls, parserState: ParserState):
errorMessage = "Expected one of these keywords: generic, port, begin, end."
token = parserState.Token
if isinstance(parserState.Token, CharacterToken):
if (token == "\n"):
parserState.NewToken = LinebreakToken(token)
parserState.NewBlock = LinebreakBlock(parserState.LastBlock,
parserState.NewToken)
parserState.TokenMarker = parserState.NewToken
return
elif (token == "-"):
parserState.PushState = SingleLineCommentBlock.statePossibleCommentStart
parserState.TokenMarker = token
return
elif (token == "/"):
parserState.PushState = MultiLineCommentBlock.statePossibleCommentStart
parserState.TokenMarker = token
return
elif isinstance(token, SpaceToken):
parserState.NewToken = IndentationToken(token)
parserState.NewBlock = IndentationBlock(parserState.LastBlock,
parserState.NewToken)
return
elif isinstance(token, StringToken):
keyword = token.Value.lower()
if (keyword == "generic"):
newToken = GenericKeyword(token)
parserState.PushState = GenericList.OpenBlock.stateGenericKeyword
elif (keyword == "port"):
newToken = PortKeyword(token)
parserState.PushState = PortList.OpenBlock.statePortKeyword
elif (keyword == "end"):
newToken = EndKeyword(token)
parserState.NextState = EndGenerateBlock.stateEndKeyword
elif (keyword == "begin"):
parserState.NewToken = BeginKeyword(token)
parserState.NewBlock = ElseGenerateBeginBlock(
parserState.LastBlock, parserState.NewToken)
parserState.NextState = ElseGenerateBeginBlock.stateBeginKeyword
return
else:
raise TokenParserException(errorMessage, token)
parserState.NewToken = newToken
parserState.TokenMarker = newToken
return
raise TokenParserException(errorMessage, token)
def stateOpeningParenthesis(cls, parserState: ParserState):
token = parserState.Token
errorMessage = "Expected generic name (identifier)."
if isinstance(token, CharacterToken):
if (token == ")"):
# if (parserState.TokenMarker != token):
# parserState.NewBlock = IndentationBlock(parserState.LastBlock, parserState.TokenMarker, token.PreviousToken)
parserState.Pop()
parserState.TokenMarker = token
return
elif (token == "\n"):
parserState.NewToken = LinebreakToken(token)
parserState.NewBlock = LinebreakBlock(parserState.LastBlock,
parserState.NewToken)
parserState.TokenMarker = None
parserState.PushState = LinebreakBlock.stateLinebreak
return
elif (token == "-"):
parserState.TokenMarker = None
parserState.PushState = SingleLineCommentBlock.statePossibleCommentStart
parserState.TokenMarker = token
return
elif (token == "/"):
parserState.TokenMarker = None
parserState.PushState = MultiLineCommentBlock.statePossibleCommentStart
parserState.TokenMarker = token
return
elif isinstance(token, SpaceToken):
parserState.NewToken = IndentationToken(token)
parserState.NewBlock = IndentationBlock(parserState.LastBlock,
parserState.NewToken)
return
elif isinstance(token, WordToken):
parserState.NewToken = IdentifierToken(token)
parserState.TokenMarker = parserState.NewToken
parserState.NextState = ItemBlock.stateItemRemainder
# if (parserState.TokenMarker != token):
# parserState.NewBlock = IndentationBlock(parserState.LastBlock, parserState.TokenMarker, token)
return
raise BlockParserException(errorMessage, token)
def GetVHDLTokenizer(cls, iterable: Iterator[str]):
previousToken = StartOfDocumentToken()
tokenKind = cls.TokenKind.OtherChars
start = SourceCodePosition(1, 1, 1)
buffer = ""
absolute = 0
column = 0
row = 1
__NUMBER_CHARACTERS__ = "0123456789"
__ALPHA_CHARACTERS__ = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
__WHITESPACE_CHARACTERS__ = " \t"
__FUSEABLE_CHARS__ = "=<:/*>?"
yield previousToken
for char in iterable:
absolute += 1
column += 1
# State: SpaceChars
if (tokenKind is cls.TokenKind.SpaceChars):
if (char in __WHITESPACE_CHARACTERS__):
buffer += char
else:
end = SourceCodePosition(row, column - 1, absolute - 1)
if isinstance(previousToken,
(LinebreakToken, SingleLineCommentToken,
StartOfDocumentToken)):
previousToken = IndentationToken(
previousToken, buffer, start, end)
else:
previousToken = SpaceToken(previousToken, buffer,
start, end)
yield previousToken
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "."):
tokenKind = cls.TokenKind.PossibleRealLiteral
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: IntegerChars
elif (tokenKind is cls.TokenKind.IntegerChars):
if ((char in __NUMBER_CHARACTERS__) or (char == "_")):
buffer += char
elif (char == "."):
buffer += char
tokenKind = cls.TokenKind.RealChars
else:
previousToken = IntegerLiteralToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: RealChars
elif (tokenKind is cls.TokenKind.RealChars):
if ((char in __NUMBER_CHARACTERS__) or (char == "_")):
buffer += char
else:
previousToken = RealLiteralToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: AlphaChars
elif (tokenKind is cls.TokenKind.AlphaChars):
if ((char in __ALPHA_CHARACTERS__) or (char == "_")):
buffer += char
else:
previousToken = WordToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: PossibleSingleLineCommentStart
elif (tokenKind is cls.TokenKind.PossibleSingleLineCommentStart):
if (char == "-"):
buffer = "--"
tokenKind = cls.TokenKind.SingleLineComment
else:
previousToken = CharacterToken(previousToken, "-", start)
yield previousToken
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "/r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "/n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: PossibleLinebreak
elif (tokenKind is cls.TokenKind.PossibleLinebreak):
end = SourceCodePosition(row, column, absolute)
if (char == "\n"):
tokenKind = cls.TokenKind.OtherChars
if (buffer[:2] == "--"):
buffer += char
previousToken = SingleLineCommentToken(
previousToken, buffer, start, end)
else:
previousToken = LinebreakToken(previousToken, "\r\n",
start, end)
buffer = "\r\n"
yield previousToken
else:
previousToken = LinebreakToken(previousToken, "\r", start,
end)
yield previousToken
start = end
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "/r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "/n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: PossibleRealLiteral
elif (tokenKind is cls.TokenKind.PossibleRealLiteral):
if (char in __NUMBER_CHARACTERS__):
buffer += char
tokenKind = cls.TokenKind.RealChars
else:
previousToken = CharacterToken(previousToken, ".", start)
yield previousToken
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "/r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "/n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: PossibleCharacterLiteral
elif (tokenKind is cls.TokenKind.PossibleCharacterLiteral):
buffer += char
if (len(buffer) == 2):
if (buffer[1] == "'"):
previousToken = CharacterToken(previousToken, "'",
start)
yield previousToken
previousToken = CharacterToken(
previousToken, "'",
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
else:
continue
elif ((len(buffer) == 3) and (buffer[2] == "'")):
previousToken = CharacterLiteralToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
else:
previousToken = CharacterToken(previousToken, "'", start)
yield previousToken
start.Column += 1
start.Absolute += 1
buffer = buffer[:2]
if ((buffer[0] in __ALPHA_CHARACTERS__)
and (buffer[1] in __ALPHA_CHARACTERS__)):
tokenKind = cls.TokenKind.AlphaChars
elif ((buffer[0] in __WHITESPACE_CHARACTERS__)
and (buffer[1] in __WHITESPACE_CHARACTERS__)):
tokenKind = cls.TokenKind.SpaceChars
else:
raise TokenizerException(
"Ambiguous syntax detected. buffer: '{buffer}'".
format(buffer=buffer), start)
# State: PossibleStringLiteralStart
elif (tokenKind is cls.TokenKind.PossibleStringLiteralStart):
buffer += char
if (char == "\""):
previousToken = StringLiteralToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: PossibleExtendedIdentifierStart
elif (tokenKind is cls.TokenKind.PossibleExtendedIdentifierStart):
buffer += char
if (char == "\\"):
previousToken = ExtendedIdentifier(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: Directive
elif (tokenKind is cls.TokenKind.Directive):
buffer += char
if (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = DirectiveToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: SingleLineComment
elif (tokenKind is cls.TokenKind.SingleLineComment):
buffer += char
if (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = SingleLineCommentToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: MultiLineComment
elif (tokenKind is cls.TokenKind.MultiLineComment):
buffer += char
if (buffer[-2:] == "*/"):
previousToken = MultiLineCommentToken(
previousToken, buffer, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
# State: FuseableCharacter
elif (tokenKind is cls.TokenKind.FuseableCharacter):
fused = buffer + char
if (fused in ("=>", "**", ":=", "/=", "<=", ">=", "<>", "<<",
">>", "??", "?=", "?<", "?>", "?/=", "?<=",
"?>=")):
previousToken = FusedCharacterToken(
previousToken, fused, start,
SourceCodePosition(row, column, absolute))
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (fused in ("?/", "?<", "?>")):
buffer = fused
elif (fused == "/*"):
buffer = fused
tokenKind = cls.TokenKind.MultiLineComment
else:
previousToken = CharacterToken(previousToken, buffer[0],
start)
yield previousToken
if (len(buffer) == 2):
previousToken = CharacterToken(previousToken,
buffer[1], start)
yield previousToken
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char,
start, start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
pass
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
previousToken = CharacterToken(previousToken, char,
start)
yield previousToken
# State: OtherChars
elif (tokenKind is cls.TokenKind.OtherChars):
start = SourceCodePosition(row, column, absolute)
buffer = char
if (char in __WHITESPACE_CHARACTERS__):
tokenKind = cls.TokenKind.SpaceChars
elif (char in __NUMBER_CHARACTERS__):
tokenKind = cls.TokenKind.IntegerChars
elif (char in __ALPHA_CHARACTERS__):
tokenKind = cls.TokenKind.AlphaChars
elif (char == "'"):
tokenKind = cls.TokenKind.PossibleCharacterLiteral
elif (char == "\""):
tokenKind = cls.TokenKind.PossibleStringLiteralStart
elif (char == "-"):
tokenKind = cls.TokenKind.PossibleSingleLineCommentStart
elif (char == "\r"):
tokenKind = cls.TokenKind.PossibleLinebreak
elif (char == "\n"):
previousToken = LinebreakToken(previousToken, char, start,
start)
yield previousToken
tokenKind = cls.TokenKind.OtherChars
elif (char in __FUSEABLE_CHARS__):
buffer = char
tokenKind = cls.TokenKind.FuseableCharacter
elif (char == "\\"):
tokenKind = cls.TokenKind.PossibleExtendedIdentifierStart
elif ((char == "`")
and isinstance(previousToken,
(SpaceToken, LinebreakToken))):
tokenKind = cls.TokenKind.Directive
else:
tokenKind = cls.TokenKind.OtherChars
previousToken = CharacterToken(previousToken, char, start)
yield previousToken
# State: unknown
else:
raise TokenizerException(
"Unknown state.",
SourceCodePosition(row, column, absolute))
if (char == "\n"):
column = 0
row += 1
# end for
if (tokenKind is cls.TokenKind.MultiLineComment):
raise TokenizerException(
"End of document before end of multi line comment.",
SourceCodePosition(row, column, absolute))
# End of document
yield EndOfDocumentToken(previousToken,
SourceCodePosition(row, column, absolute))