def buildToken(tokenString, line, colStart, colEnd): #print("BUILDING TOKEN " + tokenString) newToken = Token() newToken.name = tokenString if len(tokenString) > 31: print("WE GOT AN ERROR BOYS") newToken.hasError = True newToken.errorType = "too long" newToken.line = line newToken.colEnd = colEnd newToken.colStart = colStart if tokenString[0] in symbolChars: newToken.flavor = "\'" + tokenString + "\'" elif tokenString[0].isdigit(): if "." in tokenString: newToken.flavor = "T_DoubleConstant" else: newToken.flavor = "T_IntConstant" elif tokenString[0] == '\"' and tokenString[ len(tokenString) - 1] == '\"' and len(tokenString) > 1: newToken.flavor = "T_StringConstant" elif tokenString[0] == '\"' and (tokenString[len(tokenString) - 1] != '\"' or len(tokenString) == 1): newToken.hasError = True newToken.errorType = "unterminated string" else: newToken.flavor = "T_Identifier" return newToken
def buildUnrecognizedCharacterToken(charString, line): newToken = Token() newToken.line = line newToken.name = charString newToken.hasError = True newToken.errorType = "unrecognized char" return newToken