def addToken(self, tokenType, literal=None, indent=""): #if we haven't been given an intentation, then fetch it from memory if not indent: indent = self.indent #return the text from the sourcecode (characters between the start and current position) text = self.source[self.start:self.current] #create and return a token return Tokens.Token(tokenType, text, literal, self.line, self.char, indent)
def insertToken(self, ttype, lexeme="", literal=None, line=0, char=0, indent=0): if isinstance(ttype, str): self.tokens.insert( self.current, Tokens.Token(ttype, lexeme, literal, line, char, indent)) else: self.tokens.insert(self.current, ttype)
def scanTokens(self): #empty token list self.tokens = [] #while we are not at the end of the list, loop while not self.atEnd(): #update the position of our lexer self.char += (self.current - self.start) self.start = self.current #fetch the next token token = self.tokenFromChar() #append the token to our list of tokens, provided we recieved one if token: #if the last token was an ending if self.checkPreviousToken("End"): #fetch it lt = self.previousToken() #and check we haven't skipped an indentation if token.indent < lt.indent - 1: #create a new ending token at the same position, but with one lower indentation self.tokens.append( Tokens.Token("End", "", None, lt.line, lt.char, lt.indent - 1)) #add the token self.tokens.append(token) #add an end token if none exists if not self.checkPreviousToken("End"): self.tokens.append( Tokens.Token("End", "", None, self.line, self.char + 1, self.indent)) #add an EOF token self.tokens.append( Tokens.Token("EOF", "", None, self.line, self.char + 1, 0)) #remove any leading end tokens while self.tokens[0].type == "End": self.tokens.pop(0) return self.tokens