def opTokenBuilder(self, tempOperator): """ Parameters: * tempOperator - :rtype: a token with a value of tempOperator """ return Token.buildToken( self.token, tempOperator, 'TK_' + self.operators_KeyValue_list.get(tempOperator))
def tokenBuilder(self, tempWord, keyValue): """ Parameters: * tempWord - the value of the token * keyValue - the data type of the token :rtype: a token with a value of tempWord and a data type of keyValue """ return Token.buildToken( self.token, tempWord, keywords.defined_keywords_list.get(keyValue).upper())
class Scanner(object): # constructor def __init__(self, file): """ Parameters: * file - the pascal file to be read and tokenized :rtype: """ self.defined_keywords_list = keywords.defined_keywords_list self.supported_operators = keywords.supported_operators self.operators_KeyValue_list = keywords.operators_KeyValue_list self.array_index = 0 self.token = Token() self.token_lst = [] # read in pascal_file file with open(file, 'r') as pascal_testfile: self.pascal_file = pascal_testfile.read().lower() # ############################################################################# def tokenBuilder(self, tempWord, keyValue): """ Parameters: * tempWord - the value of the token * keyValue - the data type of the token :rtype: a token with a value of tempWord and a data type of keyValue """ return Token.buildToken( self.token, tempWord, keywords.defined_keywords_list.get(keyValue).upper()) def opTokenBuilder(self, tempOperator): """ Parameters: * tempOperator - :rtype: a token with a value of tempOperator """ return Token.buildToken( self.token, tempOperator, 'TK_' + self.operators_KeyValue_list.get(tempOperator)) # ############################################################################# # Scanner # ############################################################################# def scan(self): while len(self.pascal_file) > self.array_index: if self.pascal_file[self.array_index].isalpha(): self.token_lst.append(self.if_letter()) elif self.pascal_file[self.array_index].isdigit(): self.token_lst.append(self.if_number()) elif self.pascal_file[ self.array_index] in self.supported_operators: self.token_lst.append(self.if_operator()) elif self.pascal_file[self.array_index] == " ": self.array_index += 1 self.token.set_colNumber(self.token.get_colNumber() + 1) elif self.pascal_file[self.array_index] == "\n": self.array_index += 1 self.token.set_rowNumber(self.token.get_rowNumber() + 1) self.token.set_colNumber(0) elif self.pascal_file[self.array_index] == "\'": self.token_lst.append(self.if_quote()) else: raise TypeError("Can't identify char: " + self.pascal_file[self.array_index]) eof = self.token.buildToken( "EOF", keywords.defined_keywords_list.get("eof").upper()) self.token_lst.append(eof) return self.token_lst # ############################################################################# # Case functions follow # ############################################################################# def if_letter(self): # Parameters # Returns: A token which was returned by the helper function word_infile = "" # Loop through each character for character in self.pascal_file[self.array_index:]: # create string if character.isalpha() or character.isdigit(): word_infile += character else: return psh.help_caseLetter(self, word_infile) def if_number(self): # Parameters # Returns: A token which was returned by the helper function digit_infile = "" while self.array_index < len(self.pascal_file): # create string of numbers if self.pascal_file[self.array_index].isdigit() or \ self.pascal_file[self.array_index] == '.': digit_infile += self.pascal_file[self.array_index] self.array_index += 1 self.token.set_colNumber(self.token.get_colNumber() + 1) else: return psh.help_caseNum(self, digit_infile) def if_operator(self): # Parameters # Returns: A token which was returned by one of the helper functions tempOperator = "" while self.array_index < len(self.pascal_file): tempOperator += self.pascal_file[self.array_index] # check if single colon or assignment if self.pascal_file[self.array_index] == ":": return psh.help_caseOperator( self, tempOperator, self.pascal_file[self.array_index + 1]) # check if <, <=, or != elif self.pascal_file[self.array_index] == "<": return psh.check_lte_notequal( self, tempOperator, self.pascal_file[self.array_index + 1]) # if greater than or gte elif self.pascal_file[self.array_index] == ">": return psh.help_caseOperator( self, tempOperator, self.pascal_file[self.array_index + 1]) # check if a comment elif self.pascal_file[self.array_index] == "(": return psh.check_comment( self, tempOperator, self.pascal_file[self.array_index + 1]) else: self.array_index += 1 self.token.set_colNumber(self.token.get_colNumber() + 1) return self.opTokenBuilder(tempOperator) def if_comment(self, current, next): # Parameters # Returns: A comment token curr_comment = "" curr_comment += current + next self.array_index += 2 self.token.set_colNumber(self.token.get_colNumber() + 2) while self.array_index < len(self.pascal_file) and \ self.pascal_file[self.array_index] != "*" and self.pascal_file[self.array_index + 1] != ")": curr_comment += self.pascal_file[self.array_index] psh.help_caseComment(self) if self.array_index >= len(self.pascal_file): return "Comment never completed" curr_comment += self.pascal_file[self.array_index] + self.pascal_file[ self.array_index + 1] self.array_index += 2 self.token.set_colNumber(self.token.get_colNumber() + 2) return self.token.buildToken( curr_comment, keywords.defined_keywords_list.get("comment").upper()) def if_quote(self): # Parameters # Returns: A token of type string which was returned by the helper function built_string = "" built_string += self.pascal_file[self.array_index] self.array_index += 1 self.token.set_colNumber(self.token.get_colNumber() + 1) while self.array_index < len(self.pascal_file): psh.help_caseQuote(self, built_string, self.pascal_file[self.array_index]) # throw an error if the file ends before the string is completed if self.array_index >= len(self.pascal_file): return "String never completed"