def set_code_lines(self, code: [str]):
self.lexer = Lexer(code)
self.token = Token()
self.current_token_id = -1
self.read_tokens = 0
self.lexer_rich_the_end = 0
self.tokens = []
self.next_token()
def parse(self, filename=None):
"""
Parse the expression in file specified by filename
Filename can be specified on initialization, as well
:param filename: The filename from which the expression should be parsed
:return: The root of the built tree
"""
if filename:
self.filename = filename
self.__lexer = Lexer(filename)
if not self.filename:
print("No filename provided!")
return None
return self.__construct_tree()
from Lexer.lexer import Lexer
from Parser.yacc import Yacc
import codecs
f = codecs.open('Samples/test', encoding='utf-8')
data = f.read()
f.close()
temp = Lexer()
lexer = temp.build()
lexer.input(data)
yacc = Yacc()
parser = yacc.build().parse(data, lexer)
while True:
at = "-"
tok = lexer.token()
if not tok:
break # No more input
if tok.type == 'ID':
print(str(tok.value)+'\t'+str(tok.type)+'\t'+str(temp.state_table.index(tok.value)))
elif tok.type == 'BOOL_CONSTANT':
if tok.value == 'true':
print(str(tok.value) + '\t' + str(tok.type) + '\t' + "1")
else:
print(str(tok.value) + '\t' + str(tok.type) + '\t' + "0")
elif tok.type == 'CHAR_CONSTANT':
print(str(tok.value) + '\t' + str(tok.type) + '\t' + str(tok.value))
elif tok.type == 'INT_CONSTANT':
print(str(tok.value) + '\t' + str(tok.type) + '\t' + str(tok.value))
elif tok.type == 'FLOAT_CONSTANT':
if __name__ == '__main__':
args = sys.argv[1:]
if len(args) >= 1 and len(args) <= 3:
command = args[0]
if command == '-h':
print(
'\n-f [path to template] [path to file] - enter path to template to be used in formatting(e.g. resources/templates-settings.json") and path to file to be formatted'
)
print('-v [path to file] - enter path to file to be analyzed')
elif command == '-f':
templateName = args[1]
path = args[2]
if FileReader.isFile(path):
sourceCode = FileReader.readFile(path)
lexer = Lexer(sourceCode)
lexer.execute()
formatter = Formatter("Formatting", lexer.getTokens(), path,
templateName)
result = formatter.execute()
resultFile = FileReader.writeToFile(path, result)
print('formatted file ' + resultFile)
else:
jsFiles = FileReader.getAllJsFiles(path)
for file in jsFiles:
sourceCode = FileReader.readFile(file)
lexer = Lexer(sourceCode)
lexer.execute()
formatter = Formatter("Formatting", lexer.getTokens(),
file, templateName)
result = formatter.execute()
def __init__(self):
self.ok = True
self.lexer = Lexer()
self.parser = yacc.yacc(module=self)
self._functions = dict()
class Parser:
def set_code_lines(self, code: [str]):
self.lexer = Lexer(code)
self.token = Token()
self.current_token_id = -1
self.read_tokens = 0
self.lexer_rich_the_end = 0
self.tokens = []
self.next_token()
def next_token(self):
self.current_token_id += 1
self.token = self.get_token_at(self.current_token_id)
def get_token_at(self, pos):
if pos >= self.read_tokens and not self.lexer_rich_the_end:
for i in range(pos - self.read_tokens + 1):
self.tokens.append(self.lexer.next_token())
self.read_tokens += 1
if self.tokens[-1].type == Type.Special.EOF:
self.lexer_rich_the_end = True
break
return self.tokens[min(pos, self.read_tokens - 1)]
def is_nth(self, type_of_token, n):
if isinstance(type_of_token, list):
for type_of_cur_token in type_of_token:
if self.get_token_at(self.current_token_id +
n).type == type_of_cur_token:
return True
return False
return self.get_token_at(self.current_token_id +
n).type == type_of_token
def is_next(self, type_of_token):
if isinstance(type_of_token, list):
for type_of_cur_token in type_of_token:
if self.token.type == type_of_cur_token:
return True
return False
return self.token.type == type_of_token
def must_next(self, type_of_token):
if not self.is_next(type_of_token):
print("{0} is not {1}".format(self.token, type_of_token))
exit(1)
def __init__(self):
pass
def eat_name(self):
self.must_next(Type.Reserved.Name)
self.next_token()
self.must_next(Type.Lang.Colon)
self.next_token()
self.must_next(Type.Word)
res = self.token.value
self.next_token()
return res
def eat_protected(self):
if (self.is_next(Type.Reserved.KeyProtected)):
self.next_token()
return True
return False
def eat_magic(self):
self.must_next(Type.Reserved.Magic)
self.next_token()
self.must_next(Type.Lang.Colon)
self.next_token()
self.must_next(Type.Number.Integer)
res = self.token.value
self.next_token()
return res
def eat_params(self):
params = []
self.must_next(Type.Lang.LeftBracket)
self.next_token()
while not self.is_next(Type.Lang.RightBracket):
self.must_next(Type.Word)
typ = self.token.value
self.next_token()
self.must_next(Type.Word)
nam = self.token.value
self.next_token()
params.append([typ, nam])
if self.is_next(Type.Lang.Comma):
self.must_next(Type.Lang.Comma)
self.next_token()
self.must_next(Type.Lang.RightBracket)
self.next_token()
return params
def eat_message(self, decoder):
self.must_next(Type.Word)
msgname = self.token.value
self.next_token()
decoder.add_message(msgname, self.eat_params())
return msgname
def eat_function(self, decoder):
ms1 = self.eat_message(decoder)
ms2 = None
if self.is_next(Type.Reserved.Return):
self.must_next(Type.Reserved.Return)
self.next_token()
ms2 = self.eat_message(decoder)
decoder.add_function(ms1, ms2)
def eat_decoder(self):
self.must_next(Type.Reserved.Begin)
self.next_token()
is_protected = self.eat_protected()
decoder = Connection(self.eat_name(), self.eat_magic(), is_protected)
while not self.is_next(Type.Reserved.End):
self.eat_function(decoder)
self.must_next(Type.Reserved.End)
self.next_token()
return decoder
def parse(self):
decoders = []
while self.is_next(Type.Reserved.Begin):
decoders.append(self.eat_decoder())
print("connc: {0} decoders parsed!".format(len(decoders)))
return decoders
def __init__(self, filename=None):
self.filename = filename
if filename:
self.__lexer = Lexer(filename)
class Parser:
def __init__(self, filename=None):
self.filename = filename
if filename:
self.__lexer = Lexer(filename)
def parse(self, filename=None):
"""
Parse the expression in file specified by filename
Filename can be specified on initialization, as well
:param filename: The filename from which the expression should be parsed
:return: The root of the built tree
"""
if filename:
self.filename = filename
self.__lexer = Lexer(filename)
if not self.filename:
print("No filename provided!")
return None
return self.__construct_tree()
def __construct_tree(self):
res = self.__parse_expression()
t = self.__lexer.get()
# Expression is parsed, but EOF is not reached -> something extra in the file
if t:
raise UnexpectedTokenException(self.filename,
self.__lexer.get_position(),
t.value)
return res
def __parse_expression(self):
"""
expression -> relation
"""
return self.__parse_relation()
def __parse_relation(self):
"""
relation -> term [ ( "<" | ">" | "=" ) term ]
"""
left = self.__parse_term()
parent = self.__lexer.get()
if not parent:
return left
expected_types = (TokenType.OpEquals, TokenType.OpLessThan,
TokenType.OpMoreThan)
# Could be ")"
if parent.type not in expected_types:
self.__lexer.push_back(parent)
return left
right = self.__parse_term()
if not right:
raise UnexpectedEOF(self.filename, self.__lexer.get_position())
return self.__make_binary_tree(AstNode(parent), left, right)
def __parse_term(self):
"""
term -> factor { ( "+" | "-" ) factor }
"""
left = self.__parse_factor()
parent = self.__lexer.get()
while parent:
expected_types = (TokenType.OpPlus, TokenType.OpMinus)
if parent.type not in expected_types:
self.__lexer.push_back(parent)
break
right = self.__parse_factor()
if not right:
raise UnexpectedEOF(self.filename, self.__lexer.get_position())
left = self.__make_binary_tree(AstNode(parent), left, right)
parent = self.__lexer.get()
return left
def __parse_factor(self):
"""
factor -> primary { "*" primary }
"""
left = self.__parse_primary()
parent = self.__lexer.get()
while parent:
if parent.type is not TokenType.OpMultiply:
self.__lexer.push_back(parent)
break
right = self.__parse_primary()
if not right:
raise UnexpectedEOF(self.filename, self.__lexer.get_position())
left = self.__make_binary_tree(AstNode(parent), left, right)
parent = self.__lexer.get()
return left
def __parse_primary(self):
"""
primary -> integer | "(" expression ")"
"""
left = self.__lexer.get()
if left is None:
# Could be an empty file
return left
if left.type is TokenType.LeftParen:
left = self.__parse_expression()
closing_paren = self.__lexer.get()
if not closing_paren or closing_paren.type is not TokenType.RightParen:
raise MissingParenthesisException(self.filename,
self.__lexer.get_position())
return left
elif left.type is TokenType.Literal:
return AstNode(left)
else:
raise UnexpectedTokenException(self.filename,
self.__lexer.get_position(),
left.value)
@staticmethod
def __make_binary_tree(parent: AstNode, child1: AstNode, child2: AstNode):
"""
Create a tree stump with two children
:param parent: parent node
:param child1: first child node
:param child2: second child node
:return: root node (parent node) with specified children
"""
parent.children = [child1, child2]
child1.parent, child2.parent = parent, parent
return parent