def buildToken(text, flavor, line, colStart, colEnd, stateDeets):
newToken = Token()
newToken.text = text
newToken.flavor = flavor
newToken.line = line
newToken.colStart = colStart
newToken.colEnd = colEnd
#check for reserved ops
if newToken.text in stateDeets.reservedWords:
newToken.flavor = "T_" + str(
newToken.text[0]).upper() + newToken.text[1:]
elif newToken.text == "true" or newToken.text == "false":
newToken.flavor = "T_BoolConstant"
#check for errors
if len(newToken.text) > 31 and "Constant" not in newToken.flavor:
newToken.hasError = True
newToken.errorType = "too long"
elif newToken.flavor == "Unrecognized":
newToken.hasError = True
newToken.errorType = "unrecognized character"
elif newToken.flavor == "unterminated string":
newToken.hasError = True
newToken.errorType = "unterminated string"
stateDeets.tokenList.append(newToken)
return stateDeets
def buildUnrecognizedCharacterToken(charString, line):
newToken = Token()
newToken.line = line
newToken.name = charString
newToken.hasError = True
newToken.errorType = "unrecognized char"
return newToken
def buildToken(tokenString, line, colStart, colEnd):
#print("BUILDING TOKEN " + tokenString)
newToken = Token()
newToken.name = tokenString
if len(tokenString) > 31:
print("WE GOT AN ERROR BOYS")
newToken.hasError = True
newToken.errorType = "too long"
newToken.line = line
newToken.colEnd = colEnd
newToken.colStart = colStart
if tokenString[0] in symbolChars:
newToken.flavor = "\'" + tokenString + "\'"
elif tokenString[0].isdigit():
if "." in tokenString:
newToken.flavor = "T_DoubleConstant"
else:
newToken.flavor = "T_IntConstant"
elif tokenString[0] == '\"' and tokenString[
len(tokenString) - 1] == '\"' and len(tokenString) > 1:
newToken.flavor = "T_StringConstant"
elif tokenString[0] == '\"' and (tokenString[len(tokenString) - 1] != '\"'
or len(tokenString) == 1):
newToken.hasError = True
newToken.errorType = "unterminated string"
else:
newToken.flavor = "T_Identifier"
return newToken
def next_token(self):
found = False
while not found and not self.eof:
ch = self._getch()
if ch.isspace():
continue
self._tokenpos = self._cline + 1, self._cpos + 1
if ch in string.letters + "_":
tok = self._read_identifier(ch)
elif ch in string.digits + "$":
tok = self._read_number(ch)
elif ch == "/":
tok = self._read_comment(ch)
elif ch in "{(":
tok = self._read_block_comment(ch)
elif ch == "'":
tok = self._read_string_const(ch)
elif ch == "#":
tok = self._read_char_const(ch)
elif ch in special:
tok = self._read_delimiter(ch)
elif ch:
self.e("Illegal character '{0}'", ch)
found = ch and tok is not None
if found and ch:
tok.line, tok.pos = self._tokenpos
else:
tok = Token(tt.eof, value="EOF")
tok.line, tok.pos = self._tokenpos
if tok.line:
tok.line -= 1
else:
tok.line += 1
self._token = tok
