def buildToken(text, flavor, line, colStart, colEnd, stateDeets): newToken = Token() newToken.text = text newToken.flavor = flavor newToken.line = line newToken.colStart = colStart newToken.colEnd = colEnd #check for reserved ops if newToken.text in stateDeets.reservedWords: newToken.flavor = "T_" + str( newToken.text[0]).upper() + newToken.text[1:] elif newToken.text == "true" or newToken.text == "false": newToken.flavor = "T_BoolConstant" #check for errors if len(newToken.text) > 31 and "Constant" not in newToken.flavor: newToken.hasError = True newToken.errorType = "too long" elif newToken.flavor == "Unrecognized": newToken.hasError = True newToken.errorType = "unrecognized character" elif newToken.flavor == "unterminated string": newToken.hasError = True newToken.errorType = "unterminated string" stateDeets.tokenList.append(newToken) return stateDeets
def buildUnrecognizedCharacterToken(charString, line): newToken = Token() newToken.line = line newToken.name = charString newToken.hasError = True newToken.errorType = "unrecognized char" return newToken
def buildToken(tokenString, line, colStart, colEnd): #print("BUILDING TOKEN " + tokenString) newToken = Token() newToken.name = tokenString if len(tokenString) > 31: print("WE GOT AN ERROR BOYS") newToken.hasError = True newToken.errorType = "too long" newToken.line = line newToken.colEnd = colEnd newToken.colStart = colStart if tokenString[0] in symbolChars: newToken.flavor = "\'" + tokenString + "\'" elif tokenString[0].isdigit(): if "." in tokenString: newToken.flavor = "T_DoubleConstant" else: newToken.flavor = "T_IntConstant" elif tokenString[0] == '\"' and tokenString[ len(tokenString) - 1] == '\"' and len(tokenString) > 1: newToken.flavor = "T_StringConstant" elif tokenString[0] == '\"' and (tokenString[len(tokenString) - 1] != '\"' or len(tokenString) == 1): newToken.hasError = True newToken.errorType = "unterminated string" else: newToken.flavor = "T_Identifier" return newToken
def next_token(self): found = False while not found and not self.eof: ch = self._getch() if ch.isspace(): continue self._tokenpos = self._cline + 1, self._cpos + 1 if ch in string.letters + "_": tok = self._read_identifier(ch) elif ch in string.digits + "$": tok = self._read_number(ch) elif ch == "/": tok = self._read_comment(ch) elif ch in "{(": tok = self._read_block_comment(ch) elif ch == "'": tok = self._read_string_const(ch) elif ch == "#": tok = self._read_char_const(ch) elif ch in special: tok = self._read_delimiter(ch) elif ch: self.e("Illegal character '{0}'", ch) found = ch and tok is not None if found and ch: tok.line, tok.pos = self._tokenpos else: tok = Token(tt.eof, value="EOF") tok.line, tok.pos = self._tokenpos if tok.line: tok.line -= 1 else: tok.line += 1 self._token = tok