class TestPyPOS(unittest.TestCase): def setUp(self): '''Tests Setup''' self.lexer = Lexer() self.tagger = POSTagger() self.start = time.time() def stringTest(self,string): '''Common Testing Function''' self.words = self.lexer.lex(string) self.tags = self.tagger.tag(self.words) self.end = time.time() self.difference = self.end - self.start for tag in self.tags: print " / ".join(tag) def test_1_Short(self): '''Test Short String''' global shortTestString self.stringTest(shortTestString) def test_2_Long(self): '''Test Long String''' global testString self.stringTest(testString) def tearDown(self): print "Tokenized and tagged %s words in %s seconds" % (len(self.words),self.difference) print "Running time at test end was: %s seconds" % (time.time() - STARTTIME)
class TestPyPOS(unittest.TestCase):
def setUp(self):
'''Tests Setup'''
self.lexer = Lexer()
self.tagger = POSTagger()
self.start = time.time()
def stringTest(self, string):
'''Common Testing Function'''
self.words = self.lexer.lex(string)
self.tags = self.tagger.tag(self.words)
self.end = time.time()
self.difference = self.end - self.start
for tag in self.tags:
print " / ".join(tag)
def test_1_Short(self):
'''Test Short String'''
global shortTestString
self.stringTest(shortTestString)
def test_2_Long(self):
'''Test Long String'''
global testString
self.stringTest(testString)
def tearDown(self):
print "Tokenized and tagged %s words in %s seconds" % (len(
self.words), self.difference)
print "Running time at test end was: %s seconds" % (time.time() -
STARTTIME)
def getAST(data):
res = Lexer.lex(data)
#print (res)
ast, res = Parser.parse(res)
#print(ast.prettyPrint())
if len(res) > 0:
print("Cannot Parse: ", res)
return ast
def evaluate(self):
#lexer
lexer = Lexer(self.input)
token_list = lexer.lex()
#parser
parser = Parser(token_list)
value = parser.parse()
return value
def eval(self):
## Creating the function type
func_type = ir.FunctionType(ir.IntType(32),
[ir.IntType(32)] * len(self.argslist))
if self.name in functions_ir.keys():
raise Exception("Function with the name already exists")
func = ir.Function(self.module, func_type, self.name)
bb_entry = func.append_basic_block('entry')
self.builder = ir.IRBuilder(bb_entry)
## Create a new scope of the function
Scope(1, False).eval()
## Populate the scope with default function args
for i, arg in enumerate(func.args):
arg.name = self.argslist[i]
alloca = self.builder.alloca(ir.IntType(32), name=arg.name)
self.builder.store(arg, alloca)
variables[arg.name] = alloca
functions_ir[self.name] = func
# This part is tricky to fill the function definations ir
# in the function so that the instructions are actually filled
# in the code segment of function and not of main
text_input = function_definations[self.name]
lexer = Lexer().get_lexer()
tokens = lexer.lex(text_input)
## Since the parser fills the block of passed builder so we can't use our previous
## builder and thus we create a new parser with new builder
pg = Parser(self.module, self.builder, self.printf, self.scanf, function_definations)
pg.parse()
parser = pg.get_parser()
parser.parse(tokens).eval()
## Then we pop the scope stack and return exectution stack to previous state
Scope(-1).eval()
functions_ir[self.name] = func
from Codegener import CodeGen
from Lexer import Lexer
from Parser import Parser
with open("/home/fedor/kursach/kurs/primer.txt") as file:
text_input = file.read()
lexer = Lexer().get_lexer()
tokens = lexer.lex(text_input)
new_tokens = lexer.lex(text_input)
token_stream = []
for i in new_tokens:
token_stream.append(i)
print(i)
codegen = CodeGen()
module = codegen.module
builder = codegen.builder
printf = codegen.printf
pg = Parser(module, builder, printf)
pg.parse()
parser = pg.get_parser()
parse = parser.parse(tokens).eval()
codegen.create_ir()
codegen.save_ir("/home/fedor/kursach/kurs/out.ll")
class Parser:
def __init__(self, filename):
self.lexer = Lexer(filename)
self.advance()
def check(self, name):
return (self.currLexeme.name == name)
def advance(self):
self.currLexeme = self.lexer.lex()
def matchNoAdvance(self, name):
if not self.check(name):
print("illegal")
print("expected ", name, ", but encountered ", self.currLexeme.name, self.currLexeme.value)
exit(0)
def match(self, name):
self.matchNoAdvance(name)
temp = self.currLexeme
if self.currLexeme.name != "ENDOFFILE":
self.advance()
return temp
def operator(self):
if self.check("PLUS"):
return self.match("PLUS")
elif self.check("TIMES"):
return self.match("TIMES")
elif self.check("DIVIDE"):
return self.match("DIVIDE")
elif self.check("SUBTRACT"):
return self.match("SUBTRACT")
elif self.check("MODULUS"):
return self.match("MODULUS")
elif self.check("AND"):
return self.match("AND")
elif self.check("OR"):
return self.match("OR")
elif self.check("XOR"):
return self.match("XOR")
elif self.check("EQUALS"):
return self.match("EQUALS")
elif self.check("NOTEQUALS"):
return self.match("NOTEQUALS")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("LESSTHAN"):
return self.match("LESSTHAN")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("GREATEROREQUAL"):
return self.match("GREATEROREQUAL")
elif self.check("LESSOREQUAL"):
return self.match("LESSOREQUAL")
def operatorPending(self):
return (self.check("PLUS") or
self.check("TIMES") or
self.check("DIVIDE") or
self.check("SUBTRACT") or
self.check("MODULUS") or
self.check("AND") or
self.check("OR") or
self.check("XOR") or
self.check("EQUALS") or
self.check("NOTEQUALS") or
self.check("GREATERTHAN") or
self.check("LESSTHAN") or
self.check("GREATEROREQUAL") or
self.check("LESSOREQUAL"))
def primary(self):
if self.check("NUMBER"):
return self.match("NUMBER")
elif self.check("STRING"):
return self.match("STRING")
elif self.check("OBRACKET"):
self.match("OBRACKET")
e = self.expression()
self.match("CBRACKET")
return e
elif self.arrayPending():
return self.array()
else:
return self.varExpression()
def primaryPending(self):
return (self.check("NUMBER") or
self.check("STRING") or
self.varExpressionPending() or
self.check("OBRACKET"))
def varExpression(self):
v = self.match("VARIABLE")
if self.check("OBRACKET"):
self.match("OBRACKET")
l = self.optList()
self.match("CBRACKET")
return cons("FUNCTION_CALL", v, l)
return v
def varExpressionPending(self):
return self.check("VARIABLE")
def expression(self):
p = self.primary()
if self.operatorPending():
o = self.operator()
e = self.expression()
return cons("EXPRESSION", p, cons("GLUE", o, e))
return cons("EXPRESSION", p, None)
def expressionPending(self):
return self.primaryPending()
def list(self):
e = self.expression()
if self.check("COMMA"):
self.match("COMMA")
return cons("LIST", e, self.list())
return cons("LIST", e, None)
def optList(self):
if self.expressionPending():
return self.list()
return None
def assignment(self):
v = self.match("VARIABLE")
self.match("ASSIGN")
e = self.expression()
return cons("ASSIGN", v, e)
def array(self):
self.match("COLON")
l = self.optList()
self.match("SEMICOLON")
return cons("ARRAY", None, l)
def arrayPending(self):
return self.check("COLON")
def statement(self):
if self.ifStatementPending():
return self.ifStatement()
elif self.whileStatementPending():
return self.whileStatement()
elif self.functionDefPending():
return self.functionDef()
elif self.check("RETURN"):
self.match("RETURN")
s = self.expression()
self.match("PERIOD")
return cons("RETURN", s, None)
else:
s = self.match("VARIABLE")
if self.check("ASSIGN"):
self.match("ASSIGN")
s = cons("ASSIGN", s, self.expression())
else:
self.match("OBRACKET")
s = cons("FUNCTION_CALL", s, self.optList())
self.match("CBRACKET")
self.match("PERIOD")
return s
def statementPending(self):
return (self.ifStatementPending() or
self.whileStatementPending() or
self.functionDefPending() or
self.check("VARIABLE"))
def block(self):
if self.check("SEMICOLON"):
self.match("SEMICOLON")
else:
s = self.statement()
return cons("BLOCK", s, self.block())
return None
def ifStatement(self):
self.match("IF")
self.match("OBRACKET")
cond = self.expression()
self.match("CBRACKET")
b = self.block()
e = self.optElse()
return cons("IF", cond, cons("GLUE", b, e))
def ifStatementPending(self):
return self.check("IF")
def optElse(self):
if self.check("ELSE"):
self.match("ELSE")
b = self.block()
return cons("ELSE", b, None)
return None
def whileStatement(self):
self.match("WHILE")
self.match("OBRACKET")
cond = self.expression()
self.match("CBRACKET")
return cons("WHILE", cond, self.block())
def whileStatementPending(self):
return self.check("WHILE")
def args(self):
param = self.match("VARIABLE")
if self.check("COMMA"):
self.match("COMMA")
return cons("PARAMETERS", param, self.args())
return cons("PARAMETERS", param, None)
def optArgs(self):
if self.check("VARIABLE"):
return self.args()
return None
def functionDef(self):
self.match("FUNCTION")
name = self.match("VARIABLE")
self.match("OBRACKET")
parameters = self.optArgs()
self.match("CBRACKET")
body = self.block()
return cons("FUNCTION_DEF", name, cons("GLUE", parameters, body))
def functionDefPending(self):
return self.check("FUNCTION")
def program(self):
s = self.statement()
if self.programPending():
return cons("GLOBAL", s, self.program())
else:
return cons("GLOBAL", s, self.match("ENDOFFILE"))
def programPending(self):
return self.statementPending()
from Lexer import Lexer
from Parser import Parser
from Ast import type_name
from Eval import travel, print_tokens
import json
import pickle
filename = 'inline.c'
file = open(filename, 'r')
code = file.read()
file.close()
lexer = Lexer()
tokens, tokens_map = lexer.lex(code)
# count = 0
# for token in tokens:
# count += 1
# print(count)
# print(len(tokens_map))
#print(lexer.lex(code))
parser = Parser()
parser.get_input(tokens, tokens_map)
parser.parse()
parse_result = {'tokens':[token[1] for token in tokens], 'tokens_map':tokens_map, 'astnodes':[[node.ntype, node.content, node.subNode] for node in parser.AstNodes]}
with open('inline.pkl', 'wb') as f:
pickle.dump(parse_result, f)
#print_tokens(parser.AstNodes, 838, tokens_map, tokens)
# start, end = eval(parser.AstNodes, 425, -1, -1)
class Parser:
def __init__(self, filename):
self.lexer = Lexer(filename)
self.advance()
def check(self, name):
return (self.currLexeme.name == name)
def advance(self):
self.currLexeme = self.lexer.lex()
def matchNoAdvance(self, name):
if not self.check(name):
print("illegal")
print("expected ", name, ", but encountered ",
self.currLexeme.name, self.currLexeme.value)
exit(0)
def match(self, name):
self.matchNoAdvance(name)
temp = self.currLexeme
if self.currLexeme.name != "ENDOFFILE":
self.advance()
return temp
def operator(self):
if self.check("PLUS"):
return self.match("PLUS")
elif self.check("TIMES"):
return self.match("TIMES")
elif self.check("DIVIDE"):
return self.match("DIVIDE")
elif self.check("SUBTRACT"):
return self.match("SUBTRACT")
elif self.check("MODULUS"):
return self.match("MODULUS")
elif self.check("AND"):
return self.match("AND")
elif self.check("OR"):
return self.match("OR")
elif self.check("XOR"):
return self.match("XOR")
elif self.check("EQUALS"):
return self.match("EQUALS")
elif self.check("NOTEQUALS"):
return self.match("NOTEQUALS")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("LESSTHAN"):
return self.match("LESSTHAN")
elif self.check("GREATERTHAN"):
return self.match("GREATERTHAN")
elif self.check("GREATEROREQUAL"):
return self.match("GREATEROREQUAL")
elif self.check("LESSOREQUAL"):
return self.match("LESSOREQUAL")
def operatorPending(self):
return (self.check("PLUS") or self.check("TIMES")
or self.check("DIVIDE") or self.check("SUBTRACT")
or self.check("MODULUS") or self.check("AND")
or self.check("OR") or self.check("XOR")
or self.check("EQUALS") or self.check("NOTEQUALS")
or self.check("GREATERTHAN") or self.check("LESSTHAN")
or self.check("GREATEROREQUAL") or self.check("LESSOREQUAL"))
def primary(self):
if self.check("NUMBER"):
return self.match("NUMBER")
elif self.check("STRING"):
return self.match("STRING")
elif self.check("OBRACKET"):
self.match("OBRACKET")
e = self.expression()
self.match("CBRACKET")
return e
elif self.arrayPending():
return self.array()
else:
return self.varExpression()
def primaryPending(self):
return (self.check("NUMBER") or self.check("STRING")
or self.varExpressionPending() or self.check("OBRACKET"))
def varExpression(self):
v = self.match("VARIABLE")
if self.check("OBRACKET"):
self.match("OBRACKET")
l = self.optList()
self.match("CBRACKET")
return cons("FUNCTION_CALL", v, l)
return v
def varExpressionPending(self):
return self.check("VARIABLE")
def expression(self):
p = self.primary()
if self.operatorPending():
o = self.operator()
e = self.expression()
return cons("EXPRESSION", p, cons("GLUE", o, e))
return cons("EXPRESSION", p, None)
def expressionPending(self):
return self.primaryPending()
def list(self):
e = self.expression()
if self.check("COMMA"):
self.match("COMMA")
return cons("LIST", e, self.list())
return cons("LIST", e, None)
def optList(self):
if self.expressionPending():
return self.list()
return None
def assignment(self):
v = self.match("VARIABLE")
self.match("ASSIGN")
e = self.expression()
return cons("ASSIGN", v, e)
def array(self):
self.match("COLON")
l = self.optList()
self.match("SEMICOLON")
return cons("ARRAY", None, l)
def arrayPending(self):
return self.check("COLON")
def statement(self):
if self.ifStatementPending():
return self.ifStatement()
elif self.whileStatementPending():
return self.whileStatement()
elif self.functionDefPending():
return self.functionDef()
elif self.check("RETURN"):
self.match("RETURN")
s = self.expression()
self.match("PERIOD")
return cons("RETURN", s, None)
else:
s = self.match("VARIABLE")
if self.check("ASSIGN"):
self.match("ASSIGN")
s = cons("ASSIGN", s, self.expression())
else:
self.match("OBRACKET")
s = cons("FUNCTION_CALL", s, self.optList())
self.match("CBRACKET")
self.match("PERIOD")
return s
def statementPending(self):
return (self.ifStatementPending() or self.whileStatementPending()
or self.functionDefPending() or self.check("VARIABLE"))
def block(self):
if self.check("SEMICOLON"):
self.match("SEMICOLON")
else:
s = self.statement()
return cons("BLOCK", s, self.block())
return None
def ifStatement(self):
self.match("IF")
self.match("OBRACKET")
cond = self.expression()
self.match("CBRACKET")
b = self.block()
e = self.optElse()
return cons("IF", cond, cons("GLUE", b, e))
def ifStatementPending(self):
return self.check("IF")
def optElse(self):
if self.check("ELSE"):
self.match("ELSE")
b = self.block()
return cons("ELSE", b, None)
return None
def whileStatement(self):
self.match("WHILE")
self.match("OBRACKET")
cond = self.expression()
self.match("CBRACKET")
return cons("WHILE", cond, self.block())
def whileStatementPending(self):
return self.check("WHILE")
def args(self):
param = self.match("VARIABLE")
if self.check("COMMA"):
self.match("COMMA")
return cons("PARAMETERS", param, self.args())
return cons("PARAMETERS", param, None)
def optArgs(self):
if self.check("VARIABLE"):
return self.args()
return None
def functionDef(self):
self.match("FUNCTION")
name = self.match("VARIABLE")
self.match("OBRACKET")
parameters = self.optArgs()
self.match("CBRACKET")
body = self.block()
return cons("FUNCTION_DEF", name, cons("GLUE", parameters, body))
def functionDefPending(self):
return self.check("FUNCTION")
def program(self):
s = self.statement()
if self.programPending():
return cons("GLOBAL", s, self.program())
else:
return cons("GLOBAL", s, self.match("ENDOFFILE"))
def programPending(self):
return self.statementPending()
from Lexer import Lexer
from Parser import Parser
from Eval import Evaluator
with open("../test/function_call.ps", "r+") as f:
lexer = Lexer(f)
parser = Parser(lexer.tokens)
evaluator = Evaluator(parser.AST)
print("Tokens:")
lexer.lex()
print(lexer.tokens, '\n')
print("AST:")
parser.parse()
print(parser.AST, "\n")
print("Output:")
evaluator.run(parser.AST)
from Preprocessor import Preprocessor
from Lexer import Lexer
from Parser import Parser
text_input = """
print(4 - 4 + 2);
"""
lexer = Lexer().get_lexer()
tokens = lexer.lex(text_input)
text_input = """
a = 4 - (5 + 1);
print(a);
b = a * 2;
c = b;
print(3 + c);
print(a == b);
"""
raw_input_text = """
a = 2;
function apple(a, b, c) {
a = 2;
}
if a == 2 {
nothing; ## This to show that we are doing nothing here
}
## This is also a raw input file
else if a == 3 {
writeln(5 == 4, "Rahul", 3);
