def set_code_lines(self, code: [str]): self.lexer = Lexer(code) self.token = Token() self.current_token_id = -1 self.read_tokens = 0 self.lexer_rich_the_end = 0 self.tokens = [] self.next_token()
def parse(self, filename=None): """ Parse the expression in file specified by filename Filename can be specified on initialization, as well :param filename: The filename from which the expression should be parsed :return: The root of the built tree """ if filename: self.filename = filename self.__lexer = Lexer(filename) if not self.filename: print("No filename provided!") return None return self.__construct_tree()
from Lexer.lexer import Lexer from Parser.yacc import Yacc import codecs f = codecs.open('Samples/test', encoding='utf-8') data = f.read() f.close() temp = Lexer() lexer = temp.build() lexer.input(data) yacc = Yacc() parser = yacc.build().parse(data, lexer) while True: at = "-" tok = lexer.token() if not tok: break # No more input if tok.type == 'ID': print(str(tok.value)+'\t'+str(tok.type)+'\t'+str(temp.state_table.index(tok.value))) elif tok.type == 'BOOL_CONSTANT': if tok.value == 'true': print(str(tok.value) + '\t' + str(tok.type) + '\t' + "1") else: print(str(tok.value) + '\t' + str(tok.type) + '\t' + "0") elif tok.type == 'CHAR_CONSTANT': print(str(tok.value) + '\t' + str(tok.type) + '\t' + str(tok.value)) elif tok.type == 'INT_CONSTANT': print(str(tok.value) + '\t' + str(tok.type) + '\t' + str(tok.value)) elif tok.type == 'FLOAT_CONSTANT':
if __name__ == '__main__': args = sys.argv[1:] if len(args) >= 1 and len(args) <= 3: command = args[0] if command == '-h': print( '\n-f [path to template] [path to file] - enter path to template to be used in formatting(e.g. resources/templates-settings.json") and path to file to be formatted' ) print('-v [path to file] - enter path to file to be analyzed') elif command == '-f': templateName = args[1] path = args[2] if FileReader.isFile(path): sourceCode = FileReader.readFile(path) lexer = Lexer(sourceCode) lexer.execute() formatter = Formatter("Formatting", lexer.getTokens(), path, templateName) result = formatter.execute() resultFile = FileReader.writeToFile(path, result) print('formatted file ' + resultFile) else: jsFiles = FileReader.getAllJsFiles(path) for file in jsFiles: sourceCode = FileReader.readFile(file) lexer = Lexer(sourceCode) lexer.execute() formatter = Formatter("Formatting", lexer.getTokens(), file, templateName) result = formatter.execute()
def __init__(self): self.ok = True self.lexer = Lexer() self.parser = yacc.yacc(module=self) self._functions = dict()
class Parser: def set_code_lines(self, code: [str]): self.lexer = Lexer(code) self.token = Token() self.current_token_id = -1 self.read_tokens = 0 self.lexer_rich_the_end = 0 self.tokens = [] self.next_token() def next_token(self): self.current_token_id += 1 self.token = self.get_token_at(self.current_token_id) def get_token_at(self, pos): if pos >= self.read_tokens and not self.lexer_rich_the_end: for i in range(pos - self.read_tokens + 1): self.tokens.append(self.lexer.next_token()) self.read_tokens += 1 if self.tokens[-1].type == Type.Special.EOF: self.lexer_rich_the_end = True break return self.tokens[min(pos, self.read_tokens - 1)] def is_nth(self, type_of_token, n): if isinstance(type_of_token, list): for type_of_cur_token in type_of_token: if self.get_token_at(self.current_token_id + n).type == type_of_cur_token: return True return False return self.get_token_at(self.current_token_id + n).type == type_of_token def is_next(self, type_of_token): if isinstance(type_of_token, list): for type_of_cur_token in type_of_token: if self.token.type == type_of_cur_token: return True return False return self.token.type == type_of_token def must_next(self, type_of_token): if not self.is_next(type_of_token): print("{0} is not {1}".format(self.token, type_of_token)) exit(1) def __init__(self): pass def eat_name(self): self.must_next(Type.Reserved.Name) self.next_token() self.must_next(Type.Lang.Colon) self.next_token() self.must_next(Type.Word) res = self.token.value self.next_token() return res def eat_protected(self): if (self.is_next(Type.Reserved.KeyProtected)): self.next_token() return True return False def eat_magic(self): self.must_next(Type.Reserved.Magic) self.next_token() self.must_next(Type.Lang.Colon) self.next_token() self.must_next(Type.Number.Integer) res = self.token.value self.next_token() return res def eat_params(self): params = [] self.must_next(Type.Lang.LeftBracket) self.next_token() while not self.is_next(Type.Lang.RightBracket): self.must_next(Type.Word) typ = self.token.value self.next_token() self.must_next(Type.Word) nam = self.token.value self.next_token() params.append([typ, nam]) if self.is_next(Type.Lang.Comma): self.must_next(Type.Lang.Comma) self.next_token() self.must_next(Type.Lang.RightBracket) self.next_token() return params def eat_message(self, decoder): self.must_next(Type.Word) msgname = self.token.value self.next_token() decoder.add_message(msgname, self.eat_params()) return msgname def eat_function(self, decoder): ms1 = self.eat_message(decoder) ms2 = None if self.is_next(Type.Reserved.Return): self.must_next(Type.Reserved.Return) self.next_token() ms2 = self.eat_message(decoder) decoder.add_function(ms1, ms2) def eat_decoder(self): self.must_next(Type.Reserved.Begin) self.next_token() is_protected = self.eat_protected() decoder = Connection(self.eat_name(), self.eat_magic(), is_protected) while not self.is_next(Type.Reserved.End): self.eat_function(decoder) self.must_next(Type.Reserved.End) self.next_token() return decoder def parse(self): decoders = [] while self.is_next(Type.Reserved.Begin): decoders.append(self.eat_decoder()) print("connc: {0} decoders parsed!".format(len(decoders))) return decoders
def __init__(self, filename=None): self.filename = filename if filename: self.__lexer = Lexer(filename)
class Parser: def __init__(self, filename=None): self.filename = filename if filename: self.__lexer = Lexer(filename) def parse(self, filename=None): """ Parse the expression in file specified by filename Filename can be specified on initialization, as well :param filename: The filename from which the expression should be parsed :return: The root of the built tree """ if filename: self.filename = filename self.__lexer = Lexer(filename) if not self.filename: print("No filename provided!") return None return self.__construct_tree() def __construct_tree(self): res = self.__parse_expression() t = self.__lexer.get() # Expression is parsed, but EOF is not reached -> something extra in the file if t: raise UnexpectedTokenException(self.filename, self.__lexer.get_position(), t.value) return res def __parse_expression(self): """ expression -> relation """ return self.__parse_relation() def __parse_relation(self): """ relation -> term [ ( "<" | ">" | "=" ) term ] """ left = self.__parse_term() parent = self.__lexer.get() if not parent: return left expected_types = (TokenType.OpEquals, TokenType.OpLessThan, TokenType.OpMoreThan) # Could be ")" if parent.type not in expected_types: self.__lexer.push_back(parent) return left right = self.__parse_term() if not right: raise UnexpectedEOF(self.filename, self.__lexer.get_position()) return self.__make_binary_tree(AstNode(parent), left, right) def __parse_term(self): """ term -> factor { ( "+" | "-" ) factor } """ left = self.__parse_factor() parent = self.__lexer.get() while parent: expected_types = (TokenType.OpPlus, TokenType.OpMinus) if parent.type not in expected_types: self.__lexer.push_back(parent) break right = self.__parse_factor() if not right: raise UnexpectedEOF(self.filename, self.__lexer.get_position()) left = self.__make_binary_tree(AstNode(parent), left, right) parent = self.__lexer.get() return left def __parse_factor(self): """ factor -> primary { "*" primary } """ left = self.__parse_primary() parent = self.__lexer.get() while parent: if parent.type is not TokenType.OpMultiply: self.__lexer.push_back(parent) break right = self.__parse_primary() if not right: raise UnexpectedEOF(self.filename, self.__lexer.get_position()) left = self.__make_binary_tree(AstNode(parent), left, right) parent = self.__lexer.get() return left def __parse_primary(self): """ primary -> integer | "(" expression ")" """ left = self.__lexer.get() if left is None: # Could be an empty file return left if left.type is TokenType.LeftParen: left = self.__parse_expression() closing_paren = self.__lexer.get() if not closing_paren or closing_paren.type is not TokenType.RightParen: raise MissingParenthesisException(self.filename, self.__lexer.get_position()) return left elif left.type is TokenType.Literal: return AstNode(left) else: raise UnexpectedTokenException(self.filename, self.__lexer.get_position(), left.value) @staticmethod def __make_binary_tree(parent: AstNode, child1: AstNode, child2: AstNode): """ Create a tree stump with two children :param parent: parent node :param child1: first child node :param child2: second child node :return: root node (parent node) with specified children """ parent.children = [child1, child2] child1.parent, child2.parent = parent, parent return parent