def test_file(test_dir, filename):
cparser = Cparser()
parser = yacc.yacc(module=cparser)
try:
file_path = os.path.join(test_dir, filename)
file = open(file_path, "r")
except IOError:
print("Cannot open {0} file".format(file_path))
sys.exit(0)
old_stdout = sys.stdout
sys.stdout = mystdout = StringIO()
text = file.read()
ast = parser.parse(text, lexer=cparser.scanner, debug=False)
typeChecker = TypeChecker()
typeChecker.visit(ast)
sys.stdout = old_stdout
name = os.path.join(test_dir, os.path.splitext(filename)[0])
file_expected = "{0}.expected".format(name)
actual_content = mystdout.getvalue()
expected_content = open(file_expected).read().replace("\r", "")
res = cmp(actual_content, expected_content)
assert res == 0, "test output and file {0} differ\n---ACTUAL---\n{1}\n---EXPECTED---\n{2}\n---".format(
file_expected, actual_content, expected_content)
print("Passed test for file '%s'." % filename)
def run_pipeline(text):
ret_text = '\n' + '=================== TEXT ======================'
ret_text += '\n' + text
ret_text += '\n' + '================== TOKENS ====================='
tokens = tokenize_text(text)
ret_text += '\n' + pprint_tokens(tokens)
ret_text += '\n' + '=================== PARSE ====================='
ret_text += '\n'
parser = Serializer.load(os.getcwd() + '/parser')
#parser = LR1Parser(G)
ret_parser = parser([t.token_type for t in tokens])
parse, operations = ret_parser
if parse is None:
return ret_text + "\nParsing Error at " + operations
ret_text += '\n'.join(repr(x) for x in parse)
ret_text += '\n' + '==================== AST ======================'
ast = evaluate_reverse_parse(parse, operations, tokens)
formatter = FormatVisitor()
tree = formatter.visit(ast)
ret_text += '\n' + tree
ret_text += '\n' + '============== COLLECTING TYPES ==============='
errors = []
collector = TypeCollector(errors)
collector.visit(ast)
context = collector.context
ret_text += '\n' + 'Errors:'
for error in errors:
ret_text += '\n' + error
ret_text += '\n' + 'Context:'
ret_text += '\n' + str(context)
ret_text += '\n' + '=============== BUILDING TYPES ================'
builder = TypeBuilder(context, errors)
builder.visit(ast)
ret_text += '\n' + 'Errors: ['
for error in errors:
ret_text += '\n' + '\t'
ret_text += '\n' + error
ret_text += '\n' + ']'
ret_text += '\n' + 'Context:'
ret_text += '\n' + str(context)
ret_text += '\n' + '=============== CHECKING TYPES ================'
checker = TypeChecker(context, errors)
try:
scope, errors = checker.visit(ast)
while(checker.changed):
scope, errors = checker.visit(ast)
except SemanticError as e:
errors = [e.text]
ret_text += '\n' + 'Errors: ['
for error in errors:
ret_text += '\n' + '\t'
ret_text += '\n' + error
ret_text += '\n' + ']'
if len(errors) == 0:
checker.printer(ast)
return ret_text
def run():
filename = None
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example1"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
parser = Mparser.parser
text = file.read()
lexer.input(text)
ast = parser.parse(text, lexer=lexer)
print(ast.printTree())
typeChecker = TypeChecker()
typeChecker.visit(ast) # or alternatively ast.accept(typeChecker)
print('Checked')
ast.accept(Interpreter())
def run(filename, output=True):
file = open(filename, "r")
prsr = parser.parser
text = file.read()
ast = prsr.parse(text, lexer=scanner.lexer)
# if ast is not None:
# ast.printTree()
# print()
typeChecker = TypeChecker()
checkRes = typeChecker.visit(ast)
if output:
print(checkRes)
if isinstance(checkRes, Success):
return ast.accept(Interpreter.Interpreter(output))
import sys
import ply.yacc as yacc
from Mparser import Mparser
from TreePrinter import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example.txt"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
Mparser = Mparser()
parser = yacc.yacc(module=Mparser)
text = file.read()
ast = parser.parse(text, lexer=Mparser.scanner)
typeChecker = TypeChecker()
typeChecker.visit(ast) # or alternatively ast.accept(typeChecker)
import sys
import ply.lex as lex
import scanner # scanner.py is a file you create, (it is not an external library)
import ply.yacc as yacc
from Mparser import Mparser
from TreePrinter import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "examples/example4.txt"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
Mparser = Mparser()
parser = yacc.yacc(module=Mparser)
text = file.read()
ast = parser.parse(text, lexer=Mparser.scanner)
typeChecker = TypeChecker()
typeChecker.visit(ast)
print(typeChecker.isValid)
import os
import sys
import ply.yacc as yacc
from Cparser import Cparser
from TreePrinter import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
filename = sys.argv[1] if len(sys.argv) > 1 else "example.txt"
with open(filename, "r+") as file:
try:
_Cparser = Cparser()
parser = yacc.yacc(module=_Cparser)
text = file.read()
ast = parser.parse(text, lexer=_Cparser.scanner)
typeChecker = TypeChecker()
if ast: typeChecker.visit(ast)
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
class Parser(object):
tokens = Scanner.tokens
precedence = ( # Higher position will be executed later in case of conflicts
('right', 'ASSIGN', 'ADDASSIGN', 'SUBASSIGN', 'MULASSIGN',
'DIVASSIGN'), ('nonassoc', 'GE', 'GEQ', 'LE', 'LEQ', 'EQ',
'NEQ'), ('left', 'ADD', 'SUB', 'DOTADD', 'DOTSUB'),
('left', 'MUL', 'DIV', 'DOTMUL', 'DOTDIV'), ('left', 'TRANS'),
('right', 'UMINUS'), ('nonassoc', 'IFX'), ('nonassoc', 'ELSE'))
def __init__(self,
start="program",
outputdir="logs",
tabmodule="baseparsetab"):
create_dir(outputdir)
self.ast = False
self.type_check = False
self.scanner = Scanner()
self.type_checker = TypeChecker()
self.parser = yacc.yacc(module=self,
start=start,
tabmodule=tabmodule,
outputdir=outputdir)
self.error = False
def parse(self, text, ast=False, type_check=False):
self.ast = ast
self.type_check = type_check
self.error = False
self.parser.parse(text)
def p_error(self, p):
if p:
print(
f"Syntax error at line {p.lineno}, column {self.scanner.find_tok_column(p)}:"
f"LexToken({p.type}, {p.value})")
else:
print("Unexpected end of input")
self.error = True
def p_empty(self, p):
"""empty :"""
pass
def p_program(self, p):
"""program : statements"""
p[0] = p[1]
if self.ast:
if not self.error:
p[0].printTree()
else:
print(
f"Provided program has Syntax error, AST Tree won't be printed"
)
if self.type_check:
if not self.error:
self.type_checker.visit(
p[0]
) # Prints all Type Errors (Syntax Errors are not allowed here)
else:
print(
f"Provided program has Syntax error, Type Check won't be executed"
)
def p_statements(self, p):
"""statements : empty
| LCURLY statements RCURLY statements
| statement statements
"""
if len(p) == 2:
p[0] = Statements([])
elif len(p) == 3:
sts = p[2] if p[2] else Statements([])
sts.statements = [p[1]] + sts.statements
p[0] = sts
else:
outer = p[4] if p[4] else Statements([])
inner = p[2] if p[2] else Statements([])
outer.statements = [inner] + outer.statements
p[0] = outer
def p_block(self, p):
"""block : statement
| LCURLY statements RCURLY
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = Statements([p[2]])
def p_statement(self, p):
"""statement : IF expression block %prec IFX
| IF expression block ELSE block
| WHILE expression block
| FOR for_expression block
| PRINT expressions SEMICOL
| control_expression SEMICOL
"""
if p[1] == "if":
p[0] = If(p[2], Block(p[3])) if len(p) == 4 else If(
p[2],
Block(p[3]),
Block(p[5]),
lineno=self.scanner.lexer.lineno)
elif p[1] == "while":
p[0] = While(p[2], Block(p[3]), lineno=self.scanner.lexer.lineno)
elif p[1] == "for":
p[0] = For(p[2], Block(p[3]), lineno=self.scanner.lexer.lineno)
elif p[1] == "print":
p[0] = Print(p[2], lineno=self.scanner.lexer.lineno)
else: # control expression
p[0] = p[1]
def p_statement_assignments(self, p):
"""statement : assignments SEMICOL"""
p[0] = Assignments(p[1], lineno=self.scanner.lexer.lineno)
def p_expression(self, p):
"""expression : expression ADD expression
| expression SUB expression
| expression DIV expression
| expression MUL expression
| expression DOTADD expression
| expression DOTSUB expression
| expression DOTDIV expression
| expression DOTMUL expression
| expression GE expression
| expression GEQ expression
| expression LE expression
| expression LEQ expression
| expression EQ expression
| expression NEQ expression
"""
p[0] = BinOp(p[1], p[2], p[3], lineno=self.scanner.lexer.lineno)
def p_expression_group(self, p):
"""expression : LPAREN expression RPAREN"""
p[0] = p[2]
def p_expression_variable(self, p):
"""expression : variable"""
p[0] = p[1]
def p_variable(self, p):
"""variable : const
| matrix
"""
p[0] = Variable(p[1], lineno=self.scanner.lexer.lineno)
def p_variable_id(self, p):
"""variable : ID"""
p[0] = Variable(Id(p[1]), lineno=self.scanner.lexer.lineno)
def p_variable_uminus(self, p):
"""variable : SUB variable %prec UMINUS"""
p[0] = Variable(p[2].value,
p[2].minus + 1,
p[2].trans,
lineno=self.scanner.lexer.lineno)
def p_variable_trans(self, p):
"""variable : variable TRANS"""
p[0] = Variable(p[1].value,
p[1].minus,
p[1].trans + 1,
lineno=self.scanner.lexer.lineno)
def p_const(self, p):
"""const : STRING
| FLOATNUM
| INTNUM
"""
p[0] = p[1]
def p_for_expression(self, p):
""" for_expression : ID ASSIGN expression RANGE expression"""
p[0] = ForExpr(Id(p[1]), p[3], p[5], lineno=self.scanner.lexer.lineno)
def p_expressions(self, p):
""" expressions : expression COMMA expressions
| expression
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = [p[1]] + p[3]
def p_expressions_empty(self, p):
""" expressions : empty """
p[0] = []
def p_control_expression(self, p):
"""control_expression : BREAK
| CONTINUE
| RETURN expressions
"""
if p[1] == "break":
p[0] = Break(lineno=self.scanner.lexer.lineno)
elif p[1] == "continue":
p[0] = Continue(lineno=self.scanner.lexer.lineno)
else: # RETURN
p[0] = Return(p[2], lineno=self.scanner.lexer.lineno)
def p_assignments(self, p):
"""assignments : assignment COMMA assignments
| assignment
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = [p[1]] + p[3]
def p_assignments_empty(self, p):
"""assignments : empty """
p[0] = []
def p_assignment(self, p):
"""assignment : ID assign_op expression
| ID LBRACKET expressions RBRACKET assign_op expression
"""
if len(p) == 7:
p[0] = Assignment(Id(p[1]),
p[5],
p[6],
p[3],
lineno=self.scanner.lexer.lineno)
else:
p[0] = Assignment(Id(p[1]),
p[2],
p[3],
lineno=self.scanner.lexer.lineno)
def p_assign_op(self, p):
"""assign_op : ASSIGN
| ADDASSIGN
| SUBASSIGN
| MULASSIGN
| DIVASSIGN
"""
p[0] = p[1]
def p_matrix_special(self, p):
"""matrix : EYE LPAREN expressions RPAREN
| ZEROS LPAREN expressions RPAREN
| ONES LPAREN expressions RPAREN
"""
p[0] = SpecialMatrix(p[1], p[3], lineno=self.scanner.lexer.lineno)
def p_matrix(self, p):
"""matrix : vector"""
p[0] = SimpleMatrix(p[1], lineno=self.scanner.lexer.lineno)
def p_vector(self, p):
"""vector : LBRACKET outer_list RBRACKET"""
p[0] = p[2]
def p_outerlist(self, p):
"""outer_list : outer_list SEMICOL inner_list
| inner_list
"""
if len(p) == 2:
p[0] = p[1]
else:
p[0] = p[1] + [
';'
] + p[3] # Keep track of SEMICOL, for inputs like [1;2,3,4]
def p_innerlist(self, p):
"""inner_list : inner_list COMMA elem
| elem
"""
if len(p) == 2:
p[0] = [p[1]]
else:
p[0] = p[1] + [p[3]]
def p_innerlist_empty(self, p):
"""inner_list : empty """
p[0] = []
def p_elem(self, p):
"""elem : const
| vector
"""
p[0] = p[1]
def p_elem_id(self, p):
"""elem : ID"""
p[0] = Id(p[1], lineno=self.scanner.lexer.lineno)
from Cparser import Cparser
from TypeChecker import TypeChecker
from Interpreter import Interpreter
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example.txt"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
Cparser = Cparser()
parser = yacc.yacc(module=Cparser)
text = file.read()
ast = parser.parse(text, lexer=Cparser.scanner)
if ast:
typeChecker = TypeChecker()
typeChecker.visit(ast) # or alternatively ast.accept(typeChecker)
if typeChecker.isValid:
print "Type check finished"
ast.accept(Interpreter())
print "Interpretation finished"
else:
sys.stderr.write("Type check failed -> no interpretation")
else:
sys.stderr.write("Syntax check failed -> no type check & interpretation")
def typechecker_passes(text: str):
ast = parser.parse(text)
typechecker = TypeChecker()
typechecker.visit(ast)
return typechecker.errorok
def typechecker_fails(text: str):
ast = parser.parse(text)
typechecker = TypeChecker()
typechecker.visit(ast)
return not typechecker.errorok
import Mparser
import sys
import ply.lex as lex
import scanner
import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example1.m"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
parser = Mparser.parser
text = file.read()
lexer = scanner.lexer
t = parser.parse(text, lexer=lexer)
typeChecker = TypeChecker()
# or alternatively ast.accept(typeChecker)
for x in t:
typeChecker.visit(x)
# for x in t:
# x.printTree()
#print(t)
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(t)
import sys
import ply.yacc as yacc
from Cparser import Cparser
from TreePrinter import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example.txt"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
Cparser = Cparser()
parser = yacc.yacc(module=Cparser)
text = file.read()
ast = parser.parse(text, lexer=Cparser.scanner)
# ast.print_tree(0)
typeChecker = TypeChecker()
typeChecker.visit(ast)
parser = yacc.yacc(module=Parser)
text = file.read()
parsed = parser.parse(text, lexer=Parser.scanner)
#tr.TreePrinter.printAST(parsed)
if parsed is not None:
#print "Compiling {}".format(filename)
#print
ch = VisibilityChecker()
ch.visit(parsed)
typeChecker = TypeChecker()
typeChecker.visit(parsed)
errors = sorted(ch.errors + typeChecker.errors, key=lambda el: el.line)
warnings = sorted(ch.warnings + typeChecker.warnings, key=lambda el: el.line)
msg = "successful" if len(errors) == 0 else "failed"
for error in errors:
print error
for warning in warnings:
print warning
#print
#print "Compilation {} ({} errors, {} warnings)".format(msg, len(errors), len(warnings))
if len(errors) == 0:
#print
import sys
from ply import yacc
import scanner
import Mparser
from Interpreter import Interpreter
from TreePrinter import TreePrinter
from TypeChecker import TypeChecker
if __name__ == '__main__':
TreePrinter()
try:
filename = sys.argv[1] if len(sys.argv) > 1 else "example2.m"
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
text = file.read()
scanner = scanner.Scanner(text)
parser = Mparser.MParser(scanner)
parser = yacc.yacc(module=parser)
program = parser.parse(text, lexer=scanner.lexer)
#print(program.printTree())
typeChecker = TypeChecker()
typeChecker.visit(program) # or alternatively ast.accept(typeChecker)
if not typeChecker.errors:
program.accept(Interpreter())
import Mparser
from TreePrinter import TreePrinter
from TreeGrapher import draw_tree_from_text
from TypeChecker import TypeChecker
if __name__ == '__main__':
filename = sys.argv[1] if len(sys.argv) > 1 else "example1.m"
try:
file = open(filename, "r")
except IOError:
print("Cannot open {0} file".format(filename))
sys.exit(0)
parser = Mparser.parser
text = file.read()
ast = parser.parse(text, lexer=scanner.lexer, tracking=True)
# tree_text = ast.printTree()
# print(tree_text)
# draw_tree_from_text(tree_text, 'graph')
tc = TypeChecker()
tc.visit(ast)
if tc.error_counter == 0:
print("No errors found")
elif tc.error_counter == 1:
print("! 1 error found")
else:
print(f"! {tc.error_counter} errors found")
