def buildToken(tokenString, line, colStart, colEnd): #print("BUILDING TOKEN " + tokenString) newToken = Token() newToken.name = tokenString if len(tokenString) > 31: print("WE GOT AN ERROR BOYS") newToken.hasError = True newToken.errorType = "too long" newToken.line = line newToken.colEnd = colEnd newToken.colStart = colStart if tokenString[0] in symbolChars: newToken.flavor = "\'" + tokenString + "\'" elif tokenString[0].isdigit(): if "." in tokenString: newToken.flavor = "T_DoubleConstant" else: newToken.flavor = "T_IntConstant" elif tokenString[0] == '\"' and tokenString[ len(tokenString) - 1] == '\"' and len(tokenString) > 1: newToken.flavor = "T_StringConstant" elif tokenString[0] == '\"' and (tokenString[len(tokenString) - 1] != '\"' or len(tokenString) == 1): newToken.hasError = True newToken.errorType = "unterminated string" else: newToken.flavor = "T_Identifier" return newToken
def buildToken(text, flavor, line, colStart, colEnd, stateDeets): newToken = Token() newToken.text = text newToken.flavor = flavor newToken.line = line newToken.colStart = colStart newToken.colEnd = colEnd #check for reserved ops if newToken.text in stateDeets.reservedWords: newToken.flavor = "T_" + str( newToken.text[0]).upper() + newToken.text[1:] elif newToken.text == "true" or newToken.text == "false": newToken.flavor = "T_BoolConstant" #check for errors if len(newToken.text) > 31 and "Constant" not in newToken.flavor: newToken.hasError = True newToken.errorType = "too long" elif newToken.flavor == "Unrecognized": newToken.hasError = True newToken.errorType = "unrecognized character" elif newToken.flavor == "unterminated string": newToken.hasError = True newToken.errorType = "unterminated string" stateDeets.tokenList.append(newToken) return stateDeets
def buildUnrecognizedCharacterToken(charString, line): newToken = Token() newToken.line = line newToken.name = charString newToken.hasError = True newToken.errorType = "unrecognized char" return newToken