def parse_and_validate(host, dictionaryID, client_address,
number_of_iterations):
"""
Main method which evaluates whole code and sends respond with errors and warnings.
"""
try:
start_time = time.time() * 1000
log_to_file("START OF VALIDATION: " + str(start_time) +
", Number of iterations: " + str(number_of_iterations))
buffer = openBuffers[dictionaryID]
problems_list = []
warnings_list = []
# Decode the data
data = buffer_to_string(buffer)
lexer = python_lexer.PythonLexer()
lexer.input(data)
#for token in lexer:
# print(token.value)
lexer = python_lexer.PythonLexer()
res = python_parser.parse_data(data, lexer)
#log_time("AFTER PARSE DATA: ", time.time() * 1000, start_time)
tree, del_parts = utils.traverse_ast_test(res)
#log_time("AFTER TRAVERSE AST: ", time.time() * 1000, start_time)
ast_tree = utils.parse_with_ast(res)
#log_time("AFTER PARSE WITH AST: ", time.time() * 1000, start_time)
parser = FinalParser(number_of_iterations)
parser.eval_in_root(ast_tree)
del_lines = []
for delPart in del_parts:
for i in range(delPart[0], delPart[1] + 1):
del_lines.append(i)
#log_time("AFTER EVAL IN ROOT: ", time.time() * 1000, start_time)
#processing syntax problems
for line in del_lines:
p = []
p.append(line)
p.append('Invalid syntax.')
problems_list.append(p)
del_lines += res.emptyLinesNums
temp = {line for line in del_lines}
del_lines = [line for line in temp]
del_lines.sort()
list_of_used_lines = []
#Problems
for problem in parser.problems:
if not hasattr(problem.node, 'processed'):
problem.node.lineno = utils.getOriginLineNum(
problem.node.lineno, del_lines)
problem.node.processed = 1
if not (problem.node.lineno in list_of_used_lines):
b = []
b.append(problem.node.lineno)
b.append(str(problem))
problems_list.append(b)
list_of_used_lines.append(problem.node.lineno)
#Warnings
for warning in parser.warnings:
if not hasattr(warning.node, 'processed'):
warning.node.lineno = utils.getOriginLineNum(
warning.node.lineno, del_lines)
warning.node.processed = 1
w = []
w.append(warning.node.lineno)
w.append(str(warning))
warnings_list.append(w)
log_to_file("Problems: " + str(problems_list))
log_to_file("Warnings: " + str(warnings_list))
log_to_file("Validation completed...")
problems = json.dumps({
'problems': problems_list,
'warnings': warnings_list
})
#print("DUMPED THING: "+str(problems))
host.respond(bytes(problems, "UTF-8"), client_address)
host.validationRunning = False
log_to_file('----------------------------')
except IndentationError as error:
log_to_file("Indentation error in parsing.")
traceback.print_exc()
indent_error = json.dumps({
'message': error.msg,
'line_number': error.lineno
})
host.respond(bytes(indent_error, "UTF-8"), client_address)
host.validationRunning = False
except python_parser.RobustParserError as error:
log_to_file("Error in parsing: returning correct line number.")
b = []
b.append(error.data.lineno)
b.append("invalid syntax")
problems_list.append(b)
problems = json.dumps({
'problems': problems_list,
'warnings': warnings_list
})
host.respond(bytes(problems, "UTF-8"), client_address)
host.validationRunning = False
except Exception as error:
log_to_file("Error in parsing: ")
traceback.print_exc()
#connection.sendall(bytes("endOfValidation: "+error, "utf-8"))
#host.respond(bytes("endOfValidation", "utf-8"), client_address)
error_problems_response = json.dumps({'problems': [], 'warnings': []})
host.respond(bytes(error_problems_response, "UTF-8"), client_address)
host.validationRunning = False
def get_auto_completion(host, client_address, buffer_used, variable_line,
line_number):
'''
This method is called when user presses '.' symbol.
We want to find all possible attributes for given symbol on given buffer at given line.
Call eval_in_root plus send nodeAst - means evaluation will stop once given node is reached.
'''
try:
#Get leading spaces so the indent matches current level
variable_line = variable_line.replace('\t', ' ')
leading_spaces = len(variable_line) - len(variable_line.lstrip())
# Get only relevant part of line
line_modified = utils.getObjectStringFromLine(variable_line)
# Replace given line with string which will be resolved - much easier for our solution
resolving_name = 'target_for_completion'
buffer = buffer_used
buffer[line_number - 1] = variable_line[
0:leading_spaces] + resolving_name + ' = ' + line_modified
buffer = buffer[0:line_number]
# Parse modified buffer and eval
data = buffer_to_string(buffer)
lexer = python_lexer.PythonLexer()
lexer.input(data)
lexer = python_lexer.PythonLexer()
res = python_parser.parse_data(data, lexer)
tree, del_parts = utils.traverse_ast_test(res)
ast_tree = utils.parse_with_ast(res)
# print("TREE:"+str(utils.astNode_to_tree(ast_tree)))
del_lines = []
for delPart in del_parts:
for i in range(delPart[0], delPart[1] + 1):
del_lines.append(i)
del_lines += res.emptyLinesNums
temp = {line for line in del_lines}
del_lines = [line for line in temp]
del_lines.sort()
current_line_number = utils.getCurrentLineNum(line_number, del_lines)
parser = FinalParser(1)
parser.eval_in_root(ast_tree)
# print(str(parser.scopes[0]))
#Remove inf_ attribtues since those are used for internal purposes
list_of_all = parser.get_all_possible_attr(resolving_name)
reduced_list = []
for item in list_of_all:
if not item[0].startswith('inf_'):
reduced_list.append(item)
# Respond to the client.
response_completion = json.dumps({'options': reduced_list})
host.respond(bytes(response_completion, 'UTF-8'), client_address)
except:
traceback.print_exc()
# Send an empty list if any error occurred
list_for_completion = []
response_completion_error = json.dumps(
{'options': list_for_completion})
host.respond(bytes(response_completion_error, 'UTF-8'), client_address)
class TestInfer(InferTestCase): #{{{
def setUp(self):
self.parser = Parser()
self.finalParser = FinalParser()
self.num_class = self.parser.extern_scope['num']
self.str_class = self.parser.extern_scope['str']
self.bool_class = self.parser.extern_scope['bool']
def test_simple_parse(self):
code = makecode("""
|x = 3
|a = 'ahoj'
|y = 6
|z = 3 + x
|zz = x + y
|b = 'jozo'
|c = a + b
|x = 'mumly'
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
y = module_scope.resolve('y')
z = module_scope.resolve('z')
zz = module_scope.resolve('zz')
a = module_scope.resolve('a')
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsStr(x)
self.assertIsNum(y)
self.assertIsNum(z)
self.assertIsNum(zz)
self.assertIsStr(a)
self.assertIsStr(b)
self.assertIsStr(c)
def test_fun_parse(self):
code = makecode("""
|def mean(x,y):
| return (x+y)/2
|def gean(x,y):
| return x+y
|x = 3
|y = x+2
|z = mean(x,y)
|x = "jozo"
|zz = gean(x,"fero")
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
y = module_scope.resolve('y')
z = module_scope.resolve('z')
zz = module_scope.resolve('zz')
self.assertIsStr(x)
self.assertIsNum(y)
self.assertIsNum(z)
self.assertIsStr(zz)
def test_default_object(self):
code = makecode("""
|class A():
| pass
|a = A()
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
__setattr__ = a.resolve('__setattr__', 'straight')
self.assertEqual(__setattr__, None)
__setattr__ = a.resolve('__setattr__', mode = 'class')
self.assertEqual(__setattr__.node.name, '__setattr__')
def test_closure(self):
code = makecode("""
|def f(x):
| z = x
| def g(y):
| return(x+y)
| return g
|g1 = f(3)
|g2 = f('jozo')
|a = g1(4)
|b = g2('fero')
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
b = module_scope.resolve('b')
self.assertIsNum(a)
self.assertIsStr(b)
def test_class(self):
code = makecode("""
|class A:
| def __init__(self, x, y, z):
| self.x = x
| self.y = y
| w = 'johnie'
|
|a = A(3,"ahoj", "svet")
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
self.assert_no_problem();
self.assertIsNum(a.scope['x'])
self.assertIsStr(a.scope['y'])
self.assertNotIn('w', a.scope)
self.assertNotIn('z', a.scope)
self.assertEqual(a.scope.resolve('z'),None)
def test_override_setattr(self):
code = makecode("""
|class A:
| def __init__(self, x, y):
| pass
|
| def __setattr__(self, attr, val):
| object.__setattr__(self, attr, 4)
|
|
|a = A(3,4)
|a.x = 'jozo'
|key = 'z'
|object.__setattr__(a,key,'jozo')
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
self.assert_no_problem();
self.assertIsNum(a.scope['x'])
self.assertIsStr(a.scope['z'])
def test_method_lookup(self):
code = makecode("""
|class A:
| def __init__(self, x):
| self.x = x
| def get_x(self):
| return self.x
|a = A('jozo')
|b = a.get_x()
|a = A(3)
|c = a.get_x()
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assert_no_problem();
a = module_scope.resolve('a')
A = module_scope.resolve('A')
self.assertNotIn('get_x', a.scope)
self.assertIn('get_x', A.scope)
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsStr(b)
self.assertIsNum(c)
def test_method_manipulation(self):
code = makecode("""
|class A:
| def __init__(self, x, y):
| self.x = x
| self.y = y
|
| def fun1(self):
| return self.x+self.y
|
|
|a = A(3,4)
|a.fun1()
|fun2 = a.fun1
|a.fun3 = a.fun1
|z2 = fun2()
|z3 = a.fun3()
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
z2 = module_scope.resolve('z2')
self.assertIsNum(z2)
z3 = module_scope.resolve('z3')
self.assertIsNum(z3)
def test_class_closure(self):
code = makecode("""
|class A:
| def __init__(self, x):
| self.x = x
| def get_x(self):
| return self.x
|a = A('jozo')
|getx = a.get_x
|getx_class1 = A.get_x
|getx_class2 = A.get_x
|x1 = getx()
|x2 = getx_class1()
|x3 = getx_class2(a)
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(1, len(self.parser.problems))
self.assertEqual('getx_class1', self.parser.problems[0].symbol)
x1 = module_scope.resolve('x1')
x3 = module_scope.resolve('x3')
self.assertIsStr(x1)
self.assertIsStr(x3)
def test_inheritance(self):
code = makecode("""
|class A:
| def __init__(self):
| pass
|
| def get_x(self):
| return self.x
|class B(A):
| def __init__(self):
| pass
|
| def get_y(self):
| return self.y
|b = B()
|b.x = 'jozo'
|b.y = 4
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
b = module_scope.resolve('b')
print(b)
self.assertIsNum(b.scope['y'])
self.assertIsStr(b.scope['x'])
self.assertEqual(b.resolve('get_x', 'class').kind, 'funcdef')
def test_function_return(self):
code = makecode("""
|def g():
| return "ahoj"
|
|
|def f():
| x=1
| y=2
| g()
| return x+y;
|
|z = f()
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
z = module_scope.resolve('z')
self.assertIsNum(z)
def test_finalParser(self):
code = makecode("""
|class A:
| x = 5+1
|
|a = A()
|
|b = a.something
|
|if a.x:
| a.y=1
|else:
| a.z=1
|
|if a.x:
| a.y
|else:
| a. z
""")
node = ast.parse(code, mode = 'exec')
#self.parser.eval_in_root(node)
self.finalParser.eval_in_root(node)
#module_scope = self.parser.root_scope
#module_scope = self.finalParser.parser.root_scope
self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 2)
problem_symbols = {problem.symbol for problem in self.finalParser.problems}
self.assertEqual(problem_symbols, {'something'})
warning_symbols = {problem.symbol for problem in self.finalParser.warnings}
self.assertEqual(warning_symbols, {'y','z'})
def test_correct_inheritance(self):
code = makecode("""
|class A:
| x = 5
|
|class B:
| x = 'test'
|
|class C(A,B):
| pass
|
|class D(B,A):
| pass
|
|class E(A):
| pass
|
|class F(E,B):
| pass
|
|a = C()
|b = D()
|c = F()
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsNum(a.resolve('x', 'class'))
self.assertIsStr(b.resolve('x', 'class'))
self.assertIsNum(c.resolve('x', 'class'))
def test_isinstance_simple(self):
code = makecode("""
|class A:
| x = 4
|
|class B(A):
| y = 3
|
|class C:
| x = 2
|
|b = B()
|a = A()
|
|c = isinstance(b,A)
|d = isinstance(b,C)
|
|b.y
|b.x
|b.z
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
print("a value: "+ str(a))
c = module_scope.resolve('c')
print("c value: "+ str(c))
d = module_scope.resolve('d')
print("d value: "+ str(d))
#problem_symbols = {problem.symbol for problem in self.parser.problems}
#print("PROBLEM: "+str(problem_symbols))
def test_isinstance_expert(self):
code = makecode("""
|class A:
| x = 4
|
|class B(A):
| y = 3
|
|def returnInstanceOfA():
| a = A()
| return a
|
|def returnA():
| return A
|
|def returnB():
| return B
|
|a = A()
|b = B()
|
|c = isinstance(returnInstanceOfA(),returnA())
|d = isinstance(b,returnB())
|e = isinstance(b,returnA())
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
c = module_scope.resolve('c')
d = module_scope.resolve('d')
e = module_scope.resolve('e')
self.assertTrue(c.value)
self.assertTrue(d.value)
self.assertTrue(e.value)
#problem_symbols = {problem.symbol for problem in self.parser.problems}
#print("PROBLEM: "+str(problem_symbols))
def test_bin_ops(self):
code = makecode("""
|x = 3
|y = 5
|zz = y-x
|a = True
|b=5<6
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
zz = module_scope.resolve('zz')
a = module_scope.resolve('a')
print("zz: " + str(zz.value))
self.assertIsNum(zz)
self.assertTrue(a)
self.assertIsBool(a)
print("a: " + str(a))
b = module_scope.resolve('b')
self.assertIsBool(b)
#self.assertIsNum(a)
def test_if(self):
code = makecode("""
|c=True
|if c:
| c1=1
|v="True"
|if v:
| v1=1
|n="False"
|if n:
| n1=1
|m=False
|m1="test"
|if m:
| m1=1
|a=None
|a1="test"
|if a:
| a1=1
|s="None"
|if s:
| s1=1
|d=0
|d1="test"
|if d:
| d1=1
|f=1
|if f:
| f1=1
|h=""
|h1="test"
|if h:
| h1=1
|j="0"
|if j:
| j1=1
|k="1"
|if k:
| k1=1
|g=-1
|if g:
| g1=1
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
c1=module_scope.resolve('c1')
self.assertIsNum(c1)
v1=module_scope.resolve('v1')
self.assertIsNum(v1)
n1=module_scope.resolve('n1')
self.assertIsNum(n1)
m1=module_scope.resolve('m1')
self.assertIsStr(m1)
a1=module_scope.resolve('a1')
self.assertIsStr(a1)
s1=module_scope.resolve('s1')
self.assertIsNum(s1)
d1=module_scope.resolve('d1')
self.assertIsStr(d1)
f1=module_scope.resolve('f1')
self.assertIsNum(f1)
g1=module_scope.resolve('g1')
self.assertIsNum(g1)
h1=module_scope.resolve('h1')
self.assertIsStr(h1)
j1=module_scope.resolve('j1')
self.assertIsNum(j1)
k1=module_scope.resolve('k1')
self.assertIsNum(k1)
def test_externs(self):
code = makecode("""
|f=3.5
|x=5
|y=3
|s="retazec"
|z=x+s
|b=s+x
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
for problem in self.parser.problems:
print(problem)
self.assertEqual(problem_symbols, {'s','x'})
z=module_scope.resolve('z')
self.assertEqual(z, any_type)
x=module_scope.resolve('x')
print(str(x))
def test_obj_const(self):
code = makecode("""
|class A:
| x=4
|
|a=A()
|a.z="string"
|y=3
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
for problem in self.parser.problems:
print(problem)
#self.assertEqual(problem_symbols, {'s'})
a=module_scope.resolve('a')
self.assertIsStr(a.scope['z'])
self.assertEqual('obj', a.scope['z'].kind)
y=module_scope.resolve('y')
self.assertIsNum(y)
self.assertEqual('obj', y.kind)
def test_multiple_targets_assign(self):
code = makecode("""
|class A(object):
| def __init__(self):
| x="test"
|a=A()
|a.x=b=c=1
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a=module_scope.resolve('a')
self.assertIsNum(a.scope['x'])
b=module_scope.resolve('b')
self.assertIsNum(b)
c=module_scope.resolve('c')
self.assertIsNum(c)
def test_raise(self):
code = makecode("""
|raise 1
""")
node = ast.parse(code, mode = 'exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
self.assertEqual(problem_symbols, {'raise'})
def test_finally(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|try:
| y="test"
| doBadStuff()
| y=1
|finally:
| z=1
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y=module_scope.resolve('y')
self.assertIsStr(y)
z=module_scope.resolve('z')
self.assertIsNum(z)
self.assertEqual(len(self.parser.problems),1)
print(self.parser.problems)
def test_except(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|z=1
|x="smth"
|t="fasfas"
|try:
| y="test"
| doBadStuff()
| y=1
|except MyError:
| x=1
|else:
| t=False
|z="something"
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y=module_scope.resolve('y')
self.assertIsStr(y)
x=module_scope.resolve('x')
self.assertIsNum(x)
z=module_scope.resolve('z')
self.assertIsStr(z)
t=module_scope.resolve('t')
self.assertIsStr(t)
def test_complex_try_statement(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|class MyNextError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|def executeRiskyStuff():
| try:
| doBadStuff()
| except MyNextError:
| pass
| finally:
| pass
|
|z=1
|x="smth"
|t="fasfas"
|try:
| y="test"
| executeRiskyStuff()
| y=1
|except MyError:
| x=1
|else:
| t=False
|z="something"
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y=module_scope.resolve('y')
self.assertIsStr(y)
x=module_scope.resolve('x')
self.assertIsNum(x)
z=module_scope.resolve('z')
self.assertIsStr(z)
t=module_scope.resolve('t')
self.assertIsStr(t)
def test_list(self):
code = makecode("""
|class A:
| x=4
| def test_funkcia(self):
| print("nieco")
|
|a = A()
|l = [a,'test']
|l[0].test_funkcia()
|b = ['jozo',l]
|b[1][0].test_funkcia()
|test = t
|b[0]=1
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 2)
warning_symbols = {problem.symbol for problem in self.finalParser.warnings}
self.assertEqual(warning_symbols, {'test_funkcia'})
def test_set(self):
code = makecode("""
|set1 = {1,2,3}
|set2 = {5,6,7}
|
|
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertEqual(len(self.finalParser.problems), 2)
def test_for_and_while(self):
code = makecode("""
|class A:
| x=4
| def test_funkcia(self):
| print("nieco")
|
|a = 'jozo'
|a = t
|
|for x in range(l,5):
| c = x
| a.test_funkcia()
|
|cc = x
|
|count = 0
|while count < 10:
| print('hocico')
| print(count)
| b.test_funkcia()
|
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
#self.assertEqual(len(self.parser.problems), 2)
for problem in self.parser.problems:
print(str(problem))
module_scope = self.parser.root_scope
c = module_scope.resolve('c')
print("C: "+str(c))
def test_optional_args(self):
code = makecode("""
|class A:
| x=4
|
|def optionalArgs(a,b=1,c=True):
| print(a.x)
| return b
|
|o=A()
|x=optionalArgs(o)
|y=optionalArgs(o,'test')
|z=optionalArgs(o,3,5)
|u=optionalArgs(o,3,5,9)
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
self.assertIsNum(x)
y = module_scope.resolve('y')
self.assertIsStr(y)
for problem in self.parser.problems:
print(str(problem))
self.assertEqual(len(self.parser.problems), 1)
def test_keyword_args(self):
code = makecode("""
|class A:
| x=4
|
|def func(a,b,c):
| print(a.x)
| print(b.x)
| print(c)
|
|z=A()
|func(z,c=3,b=z)
|func(z,c=3,a=1)
|func(a=z,c=1,d=3)
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
def test_complex_args(self):
code = makecode("""
|class A:
| def __init__(self,x,y=1,z=True):
| self.x=x
| self.y=y
| self.z=z
|
|a=A("test",z=1,y=True)
|x=a.x
|y=a.y
|z=a.z
|
|b=A("test")
|x1=b.x
|y1=b.y
|z1=b.z
|
|c=A("test",z=1)
|x2=c.x
|y2=c.y
|z2=c.z
|
|d=A(z=1,b=2,y=3)
|e=A(1,z=1,x=True)
|f=A(1,2,3,4)
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
self.assertIsStr(x)
y = module_scope.resolve('y')
self.assertIsBool(y)
z = module_scope.resolve('z')
self.assertIsNum(z)
x1 = module_scope.resolve('x1')
self.assertIsStr(x1)
y1 = module_scope.resolve('y1')
self.assertIsNum(y1)
z1 = module_scope.resolve('z1')
self.assertIsBool(z1)
x2 = module_scope.resolve('x2')
self.assertIsStr(x2)
y2 = module_scope.resolve('y2')
self.assertIsNum(y2)
z2 = module_scope.resolve('z2')
self.assertIsNum(z2)
for problem in self.parser.problems:
print(str(problem))
self.assertEqual(len(self.parser.problems), 3)
def test_bool_ops(self):
code = makecode("""
|x=8
|y=False
|z=True
|u=1568
|res=False
|if x and y or u and z:
| res=True
|else:
| res="test"
|
|if y or z:
| x="test"
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
res=module_scope.resolve('res')
self.assertIsBool(res)
x=module_scope.resolve('x')
self.assertIsStr(x)
def test_dict(self):
code = makecode("""
|class A(object):
| def __init__(self):
| self.x = 5
|a=A()
|d={1:"test1"}
|d[1] = a
|print(d[1].x)
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
#self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 1)
warning_symbols = {problem.symbol for problem in self.finalParser.warnings}
self.assertEqual(warning_symbols, {'x'})
def test_set_repr(self):
code = makecode("""
|s={1,2,3}
|x=s.pop()
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
x = module_scope.resolve('x')
self.assertIsNum(x)
def test_set_ops(self):
code = makecode("""
|class A(object):
| def __init__(self):
| self.x=4
|s={1,2,3}
|t={2,3,4}
|z= s and t
|z.pop()
|a=A()
|z.add(a)
|b=z.pop()
|b.x
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 0)
self.assertEqual(len(self.finalParser.warnings), 1)
warning_symbols = {problem.symbol for problem in self.finalParser.warnings}
self.assertEqual(warning_symbols, {'x'})
def test_unary_ops(self):
code = makecode("""
|x=1
|y=-x
|z=+x
|u=~x
|w=not x
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
y = module_scope.resolve('y')
self.assertIsNum(y)
z = module_scope.resolve('z')
self.assertIsNum(z)
u = module_scope.resolve('u')
self.assertIsNum(u)
w= module_scope.resolve('w')
self.assertIsBool(w)
print(w)
def test_num_equal(self):
code = makecode("""
|x=1
|y=3
|z="test"
|if y==x:
| z=2
|else:
| z=False
|v=1
|if x==1:
| v="test"
|w=x+y
|if w==4:
| w=p
|else:
| w=q
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 2)
self.assertEqual(len(self.finalParser.warnings), 0)
problem_symbols = {problem.symbol for problem in self.finalParser.problems}
self.assertEqual(problem_symbols, {'p','q'})
def test_tuple(self):
code = makecode("""
|t = (1,2,3,4)
|s = ("TEST",)
|v = [5]
|u = t+s
|x = u[2]
|m = u[4]
|y = u[0:3]
|z = t+v
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
x = module_scope.resolve('x')
self.assertIsNum(x)
print("x: " + str(x))
m = module_scope.resolve('m')
print("m: "+ str(m))
self.assertIsStr(m)
#z = module_scope.resolve('z')
#self.assertIsNum(z)
#u = module_scope.resolve('u')
#self.assertIsNum(u)
#w= module_scope.resolve('w')
#self.assertIsBool(w)
#print(w)
def test_scope_merging(self):
code = makecode("""
|class A:
| def __init__(self):
| self.x=4
|
|c=5+6
|a=A()
|if c:
| a.y=1
|else:
| a.z=2
|
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 0)
self.assertEqual(len(self.finalParser.warnings), 0)
#module_scope=self.finalParser.finalScope
a_atrs = self.finalParser.get_all_possible_attr('a')
self.assertEqual(len(a_atrs),8)
def test_break_continue(self):
code = makecode("""
|z="test"
|for x in [1,2,3,4]:
| print(x)
| y=2
| break
| z=2
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
y = module_scope.resolve('y')
self.assertIsNum(y)
z = module_scope.resolve('z')
self.assertIsStr(z)
def test_inc_dec(self):
code = makecode("""
|class A:
| def __init__(self):
| self.x = 4
|x=1
|z=2
|x+=z
|a=A()
|a.x+=1
""")
node=ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
x = module_scope.resolve('x')
print("x"+str(x))
a = module_scope.resolve('a')
self.assertIsNum(a.scope['x'])
print("a.x: " + str(a.scope['x']))
def test_complex_example(self):
code = makecode("""
|class EmptyWareHouseException:
| pass
|class FullWareHouseException(Exception):
| pass
|class Farm:
| def __init__(self,name,size):
| self.name = name
| self.sizeOfWareHouse = size
| self.currentSize = 0
| def produceOneUnit(self):
| if self.currentSize==self.sizeOfWareHouse:
| raise FullWareHouseException()
| self.currentSize += 1
| def consumeOneUnit(self):
| if self.currentSize==0:
| raise EmptyWareHouseException()
| self.currentSize -= 1
|
|farm = Farm(size=50,name='Farm from wonderland')
|farm.produceOneUnit()
|farm.consumeOneUnit()
|farm.produceOneUnit()
""")
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
print("P:" +str(self.finalParser.problems))
print("W:" +str(self.finalParser.warnings))
#self.assertTrue(self.finalParser.nondet)
#self.assertEqual(len(self.finalParser.problems), 2)
#self.assertEqual(len(self.finalParser.warnings), 0)
#problem_symbols = {problem.symbol for problem in self.finalParser.problems}
#self.assertEqual(problem_symbols, {'p','q'})
def test_complex_example2(self):
code = makecode('''
|class Element(object):
| def __init__(self):
| self.data = 10
| def printX(self):
| print(self.x) #error
|
|instanceOfElement = Element()
|testList = [instanceOfElement, "test"]
|testList[0].printX()
|10 + testList[1] #error
''')
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
print("P:" +str(self.finalParser.problems))
print("W:" +str(self.finalParser.warnings))
def test_tuple_assign(self):
code = makecode('''
|def makePair(x,y):
| return (x,y)
|a,b = makePair(5,'five')
''')
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
def test_help(self):
code = makecode('''
|def func():
| """
| vnjvndfkvns cnscksdcnsj
| """
| x=4
|i[3]
''')
node = ast.parse(code, mode = 'exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
f = module_scope.resolve('func')
if hasattr(f,'docstring'):
print(f.docstring)
def setUp(self):
self.parser = Parser()
self.finalParser = FinalParser()
self.num_class = self.parser.extern_scope['num']
self.str_class = self.parser.extern_scope['str']
self.bool_class = self.parser.extern_scope['bool']
def setUp(self):
self.parser = Parser()
self.finalParser = FinalParser()
self.num_class = self.parser.extern_scope['num']
self.str_class = self.parser.extern_scope['str']
self.bool_class = self.parser.extern_scope['bool']
class TestInfer(InferTestCase): #{{{
def setUp(self):
self.parser = Parser()
self.finalParser = FinalParser()
self.num_class = self.parser.extern_scope['num']
self.str_class = self.parser.extern_scope['str']
self.bool_class = self.parser.extern_scope['bool']
def test_simple_parse(self):
code = makecode("""
|x = 3
|a = 'ahoj'
|y = 6
|z = 3 + x
|zz = x + y
|b = 'jozo'
|c = a + b
|x = 'mumly'
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
y = module_scope.resolve('y')
z = module_scope.resolve('z')
zz = module_scope.resolve('zz')
a = module_scope.resolve('a')
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsStr(x)
self.assertIsNum(y)
self.assertIsNum(z)
self.assertIsNum(zz)
self.assertIsStr(a)
self.assertIsStr(b)
self.assertIsStr(c)
def test_fun_parse(self):
code = makecode("""
|def mean(x,y):
| return (x+y)/2
|def gean(x,y):
| return x+y
|x = 3
|y = x+2
|z = mean(x,y)
|x = "jozo"
|zz = gean(x,"fero")
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
y = module_scope.resolve('y')
z = module_scope.resolve('z')
zz = module_scope.resolve('zz')
self.assertIsStr(x)
self.assertIsNum(y)
self.assertIsNum(z)
self.assertIsStr(zz)
def test_default_object(self):
code = makecode("""
|class A():
| pass
|a = A()
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
__setattr__ = a.resolve('__setattr__', 'straight')
self.assertEqual(__setattr__, None)
__setattr__ = a.resolve('__setattr__', mode='class')
self.assertEqual(__setattr__.node.name, '__setattr__')
def test_closure(self):
code = makecode("""
|def f(x):
| z = x
| def g(y):
| return(x+y)
| return g
|g1 = f(3)
|g2 = f('jozo')
|a = g1(4)
|b = g2('fero')
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
b = module_scope.resolve('b')
self.assertIsNum(a)
self.assertIsStr(b)
def test_class(self):
code = makecode("""
|class A:
| def __init__(self, x, y, z):
| self.x = x
| self.y = y
| w = 'johnie'
|
|a = A(3,"ahoj", "svet")
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
self.assert_no_problem()
self.assertIsNum(a.scope['x'])
self.assertIsStr(a.scope['y'])
self.assertNotIn('w', a.scope)
self.assertNotIn('z', a.scope)
self.assertEqual(a.scope.resolve('z'), None)
def test_override_setattr(self):
code = makecode("""
|class A:
| def __init__(self, x, y):
| pass
|
| def __setattr__(self, attr, val):
| object.__setattr__(self, attr, 4)
|
|
|a = A(3,4)
|a.x = 'jozo'
|key = 'z'
|object.__setattr__(a,key,'jozo')
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
self.assert_no_problem()
self.assertIsNum(a.scope['x'])
self.assertIsStr(a.scope['z'])
def test_method_lookup(self):
code = makecode("""
|class A:
| def __init__(self, x):
| self.x = x
| def get_x(self):
| return self.x
|a = A('jozo')
|b = a.get_x()
|a = A(3)
|c = a.get_x()
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assert_no_problem()
a = module_scope.resolve('a')
A = module_scope.resolve('A')
self.assertNotIn('get_x', a.scope)
self.assertIn('get_x', A.scope)
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsStr(b)
self.assertIsNum(c)
def test_method_manipulation(self):
code = makecode("""
|class A:
| def __init__(self, x, y):
| self.x = x
| self.y = y
|
| def fun1(self):
| return self.x+self.y
|
|
|a = A(3,4)
|a.fun1()
|fun2 = a.fun1
|a.fun3 = a.fun1
|z2 = fun2()
|z3 = a.fun3()
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
z2 = module_scope.resolve('z2')
self.assertIsNum(z2)
z3 = module_scope.resolve('z3')
self.assertIsNum(z3)
def test_class_closure(self):
code = makecode("""
|class A:
| def __init__(self, x):
| self.x = x
| def get_x(self):
| return self.x
|a = A('jozo')
|getx = a.get_x
|getx_class1 = A.get_x
|getx_class2 = A.get_x
|x1 = getx()
|x2 = getx_class1()
|x3 = getx_class2(a)
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(1, len(self.parser.problems))
self.assertEqual('getx_class1', self.parser.problems[0].symbol)
x1 = module_scope.resolve('x1')
x3 = module_scope.resolve('x3')
self.assertIsStr(x1)
self.assertIsStr(x3)
def test_inheritance(self):
code = makecode("""
|class A:
| def __init__(self):
| pass
|
| def get_x(self):
| return self.x
|class B(A):
| def __init__(self):
| pass
|
| def get_y(self):
| return self.y
|b = B()
|b.x = 'jozo'
|b.y = 4
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
b = module_scope.resolve('b')
print(b)
self.assertIsNum(b.scope['y'])
self.assertIsStr(b.scope['x'])
self.assertEqual(b.resolve('get_x', 'class').kind, 'funcdef')
def test_function_return(self):
code = makecode("""
|def g():
| return "ahoj"
|
|
|def f():
| x=1
| y=2
| g()
| return x+y;
|
|z = f()
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
z = module_scope.resolve('z')
self.assertIsNum(z)
def test_finalParser(self):
code = makecode("""
|class A:
| x = 5+1
|
|a = A()
|
|b = a.something
|
|if a.x:
| a.y=1
|else:
| a.z=1
|
|if a.x:
| a.y
|else:
| a. z
""")
node = ast.parse(code, mode='exec')
#self.parser.eval_in_root(node)
self.finalParser.eval_in_root(node)
#module_scope = self.parser.root_scope
#module_scope = self.finalParser.parser.root_scope
self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 2)
problem_symbols = {
problem.symbol
for problem in self.finalParser.problems
}
self.assertEqual(problem_symbols, {'something'})
warning_symbols = {
problem.symbol
for problem in self.finalParser.warnings
}
self.assertEqual(warning_symbols, {'y', 'z'})
def test_correct_inheritance(self):
code = makecode("""
|class A:
| x = 5
|
|class B:
| x = 'test'
|
|class C(A,B):
| pass
|
|class D(B,A):
| pass
|
|class E(A):
| pass
|
|class F(E,B):
| pass
|
|a = C()
|b = D()
|c = F()
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
b = module_scope.resolve('b')
c = module_scope.resolve('c')
self.assertIsNum(a.resolve('x', 'class'))
self.assertIsStr(b.resolve('x', 'class'))
self.assertIsNum(c.resolve('x', 'class'))
def test_isinstance_simple(self):
code = makecode("""
|class A:
| x = 4
|
|class B(A):
| y = 3
|
|class C:
| x = 2
|
|b = B()
|a = A()
|
|c = isinstance(b,A)
|d = isinstance(b,C)
|
|b.y
|b.x
|b.z
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
print("a value: " + str(a))
c = module_scope.resolve('c')
print("c value: " + str(c))
d = module_scope.resolve('d')
print("d value: " + str(d))
#problem_symbols = {problem.symbol for problem in self.parser.problems}
#print("PROBLEM: "+str(problem_symbols))
def test_isinstance_expert(self):
code = makecode("""
|class A:
| x = 4
|
|class B(A):
| y = 3
|
|def returnInstanceOfA():
| a = A()
| return a
|
|def returnA():
| return A
|
|def returnB():
| return B
|
|a = A()
|b = B()
|
|c = isinstance(returnInstanceOfA(),returnA())
|d = isinstance(b,returnB())
|e = isinstance(b,returnA())
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
c = module_scope.resolve('c')
d = module_scope.resolve('d')
e = module_scope.resolve('e')
self.assertTrue(c.value)
self.assertTrue(d.value)
self.assertTrue(e.value)
#problem_symbols = {problem.symbol for problem in self.parser.problems}
#print("PROBLEM: "+str(problem_symbols))
def test_bin_ops(self):
code = makecode("""
|x = 3
|y = 5
|zz = y-x
|a = True
|b=5<6
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
zz = module_scope.resolve('zz')
a = module_scope.resolve('a')
print("zz: " + str(zz.value))
self.assertIsNum(zz)
self.assertTrue(a)
self.assertIsBool(a)
print("a: " + str(a))
b = module_scope.resolve('b')
self.assertIsBool(b)
#self.assertIsNum(a)
def test_if(self):
code = makecode("""
|c=True
|if c:
| c1=1
|v="True"
|if v:
| v1=1
|n="False"
|if n:
| n1=1
|m=False
|m1="test"
|if m:
| m1=1
|a=None
|a1="test"
|if a:
| a1=1
|s="None"
|if s:
| s1=1
|d=0
|d1="test"
|if d:
| d1=1
|f=1
|if f:
| f1=1
|h=""
|h1="test"
|if h:
| h1=1
|j="0"
|if j:
| j1=1
|k="1"
|if k:
| k1=1
|g=-1
|if g:
| g1=1
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
c1 = module_scope.resolve('c1')
self.assertIsNum(c1)
v1 = module_scope.resolve('v1')
self.assertIsNum(v1)
n1 = module_scope.resolve('n1')
self.assertIsNum(n1)
m1 = module_scope.resolve('m1')
self.assertIsStr(m1)
a1 = module_scope.resolve('a1')
self.assertIsStr(a1)
s1 = module_scope.resolve('s1')
self.assertIsNum(s1)
d1 = module_scope.resolve('d1')
self.assertIsStr(d1)
f1 = module_scope.resolve('f1')
self.assertIsNum(f1)
g1 = module_scope.resolve('g1')
self.assertIsNum(g1)
h1 = module_scope.resolve('h1')
self.assertIsStr(h1)
j1 = module_scope.resolve('j1')
self.assertIsNum(j1)
k1 = module_scope.resolve('k1')
self.assertIsNum(k1)
def test_externs(self):
code = makecode("""
|f=3.5
|x=5
|y=3
|s="retazec"
|z=x+s
|b=s+x
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
for problem in self.parser.problems:
print(problem)
self.assertEqual(problem_symbols, {'s', 'x'})
z = module_scope.resolve('z')
self.assertEqual(z, any_type)
x = module_scope.resolve('x')
print(str(x))
def test_obj_const(self):
code = makecode("""
|class A:
| x=4
|
|a=A()
|a.z="string"
|y=3
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
for problem in self.parser.problems:
print(problem)
#self.assertEqual(problem_symbols, {'s'})
a = module_scope.resolve('a')
self.assertIsStr(a.scope['z'])
self.assertEqual('obj', a.scope['z'].kind)
y = module_scope.resolve('y')
self.assertIsNum(y)
self.assertEqual('obj', y.kind)
def test_multiple_targets_assign(self):
code = makecode("""
|class A(object):
| def __init__(self):
| x="test"
|a=A()
|a.x=b=c=1
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
a = module_scope.resolve('a')
self.assertIsNum(a.scope['x'])
b = module_scope.resolve('b')
self.assertIsNum(b)
c = module_scope.resolve('c')
self.assertIsNum(c)
def test_raise(self):
code = makecode("""
|raise 1
""")
node = ast.parse(code, mode='exec')
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
problem_symbols = {problem.symbol for problem in self.parser.problems}
self.assertEqual(problem_symbols, {'raise'})
def test_finally(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|try:
| y="test"
| doBadStuff()
| y=1
|finally:
| z=1
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y = module_scope.resolve('y')
self.assertIsStr(y)
z = module_scope.resolve('z')
self.assertIsNum(z)
self.assertEqual(len(self.parser.problems), 1)
print(self.parser.problems)
def test_except(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|z=1
|x="smth"
|t="fasfas"
|try:
| y="test"
| doBadStuff()
| y=1
|except MyError:
| x=1
|else:
| t=False
|z="something"
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y = module_scope.resolve('y')
self.assertIsStr(y)
x = module_scope.resolve('x')
self.assertIsNum(x)
z = module_scope.resolve('z')
self.assertIsStr(z)
t = module_scope.resolve('t')
self.assertIsStr(t)
def test_complex_try_statement(self):
code = makecode("""
|class MyError(Exception):
| pass
|
|class MyNextError(Exception):
| pass
|
|def doBadStuff():
| raise MyError()
|
|def executeRiskyStuff():
| try:
| doBadStuff()
| except MyNextError:
| pass
| finally:
| pass
|
|z=1
|x="smth"
|t="fasfas"
|try:
| y="test"
| executeRiskyStuff()
| y=1
|except MyError:
| x=1
|else:
| t=False
|z="something"
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
y = module_scope.resolve('y')
self.assertIsStr(y)
x = module_scope.resolve('x')
self.assertIsNum(x)
z = module_scope.resolve('z')
self.assertIsStr(z)
t = module_scope.resolve('t')
self.assertIsStr(t)
def test_list(self):
code = makecode("""
|class A:
| x=4
| def test_funkcia(self):
| print("nieco")
|
|a = A()
|l = [a,'test']
|l[0].test_funkcia()
|b = ['jozo',l]
|b[1][0].test_funkcia()
|test = t
|b[0]=1
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 2)
warning_symbols = {
problem.symbol
for problem in self.finalParser.warnings
}
self.assertEqual(warning_symbols, {'test_funkcia'})
def test_set(self):
code = makecode("""
|set1 = {1,2,3}
|set2 = {5,6,7}
|
|
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertEqual(len(self.finalParser.problems), 2)
def test_for_and_while(self):
code = makecode("""
|class A:
| x=4
| def test_funkcia(self):
| print("nieco")
|
|a = 'jozo'
|a = t
|
|for x in range(l,5):
| c = x
| a.test_funkcia()
|
|cc = x
|
|count = 0
|while count < 10:
| print('hocico')
| print(count)
| b.test_funkcia()
|
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
#self.assertEqual(len(self.parser.problems), 2)
for problem in self.parser.problems:
print(str(problem))
module_scope = self.parser.root_scope
c = module_scope.resolve('c')
print("C: " + str(c))
def test_optional_args(self):
code = makecode("""
|class A:
| x=4
|
|def optionalArgs(a,b=1,c=True):
| print(a.x)
| return b
|
|o=A()
|x=optionalArgs(o)
|y=optionalArgs(o,'test')
|z=optionalArgs(o,3,5)
|u=optionalArgs(o,3,5,9)
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
self.assertIsNum(x)
y = module_scope.resolve('y')
self.assertIsStr(y)
for problem in self.parser.problems:
print(str(problem))
self.assertEqual(len(self.parser.problems), 1)
def test_keyword_args(self):
code = makecode("""
|class A:
| x=4
|
|def func(a,b,c):
| print(a.x)
| print(b.x)
| print(c)
|
|z=A()
|func(z,c=3,b=z)
|func(z,c=3,a=1)
|func(a=z,c=1,d=3)
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
def test_complex_args(self):
code = makecode("""
|class A:
| def __init__(self,x,y=1,z=True):
| self.x=x
| self.y=y
| self.z=z
|
|a=A("test",z=1,y=True)
|x=a.x
|y=a.y
|z=a.z
|
|b=A("test")
|x1=b.x
|y1=b.y
|z1=b.z
|
|c=A("test",z=1)
|x2=c.x
|y2=c.y
|z2=c.z
|
|d=A(z=1,b=2,y=3)
|e=A(1,z=1,x=True)
|f=A(1,2,3,4)
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
x = module_scope.resolve('x')
self.assertIsStr(x)
y = module_scope.resolve('y')
self.assertIsBool(y)
z = module_scope.resolve('z')
self.assertIsNum(z)
x1 = module_scope.resolve('x1')
self.assertIsStr(x1)
y1 = module_scope.resolve('y1')
self.assertIsNum(y1)
z1 = module_scope.resolve('z1')
self.assertIsBool(z1)
x2 = module_scope.resolve('x2')
self.assertIsStr(x2)
y2 = module_scope.resolve('y2')
self.assertIsNum(y2)
z2 = module_scope.resolve('z2')
self.assertIsNum(z2)
for problem in self.parser.problems:
print(str(problem))
self.assertEqual(len(self.parser.problems), 3)
def test_bool_ops(self):
code = makecode("""
|x=8
|y=False
|z=True
|u=1568
|res=False
|if x and y or u and z:
| res=True
|else:
| res="test"
|
|if y or z:
| x="test"
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
res = module_scope.resolve('res')
self.assertIsBool(res)
x = module_scope.resolve('x')
self.assertIsStr(x)
def test_dict(self):
code = makecode("""
|class A(object):
| def __init__(self):
| self.x = 5
|a=A()
|d={1:"test1"}
|d[1] = a
|print(d[1].x)
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
#self.assertEqual(len(self.finalParser.problems), 1)
self.assertEqual(len(self.finalParser.warnings), 1)
warning_symbols = {
problem.symbol
for problem in self.finalParser.warnings
}
self.assertEqual(warning_symbols, {'x'})
def test_set_repr(self):
code = makecode("""
|s={1,2,3}
|x=s.pop()
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
self.assertEqual(len(self.parser.problems), 0)
x = module_scope.resolve('x')
self.assertIsNum(x)
def test_set_ops(self):
code = makecode("""
|class A(object):
| def __init__(self):
| self.x=4
|s={1,2,3}
|t={2,3,4}
|z= s and t
|z.pop()
|a=A()
|z.add(a)
|b=z.pop()
|b.x
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 0)
self.assertEqual(len(self.finalParser.warnings), 1)
warning_symbols = {
problem.symbol
for problem in self.finalParser.warnings
}
self.assertEqual(warning_symbols, {'x'})
def test_unary_ops(self):
code = makecode("""
|x=1
|y=-x
|z=+x
|u=~x
|w=not x
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
y = module_scope.resolve('y')
self.assertIsNum(y)
z = module_scope.resolve('z')
self.assertIsNum(z)
u = module_scope.resolve('u')
self.assertIsNum(u)
w = module_scope.resolve('w')
self.assertIsBool(w)
print(w)
def test_num_equal(self):
code = makecode("""
|x=1
|y=3
|z="test"
|if y==x:
| z=2
|else:
| z=False
|v=1
|if x==1:
| v="test"
|w=x+y
|if w==4:
| w=p
|else:
| w=q
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 2)
self.assertEqual(len(self.finalParser.warnings), 0)
problem_symbols = {
problem.symbol
for problem in self.finalParser.problems
}
self.assertEqual(problem_symbols, {'p', 'q'})
def test_tuple(self):
code = makecode("""
|t = (1,2,3,4)
|s = ("TEST",)
|v = [5]
|u = t+s
|x = u[2]
|m = u[4]
|y = u[0:3]
|z = t+v
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
x = module_scope.resolve('x')
self.assertIsNum(x)
print("x: " + str(x))
m = module_scope.resolve('m')
print("m: " + str(m))
self.assertIsStr(m)
#z = module_scope.resolve('z')
#self.assertIsNum(z)
#u = module_scope.resolve('u')
#self.assertIsNum(u)
#w= module_scope.resolve('w')
#self.assertIsBool(w)
#print(w)
def test_scope_merging(self):
code = makecode("""
|class A:
| def __init__(self):
| self.x=4
|
|c=5+6
|a=A()
|if c:
| a.y=1
|else:
| a.z=2
|
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
#self.assertTrue(self.finalParser.nondet)
self.assertEqual(len(self.finalParser.problems), 0)
self.assertEqual(len(self.finalParser.warnings), 0)
#module_scope=self.finalParser.finalScope
a_atrs = self.finalParser.get_all_possible_attr('a')
self.assertEqual(len(a_atrs), 8)
def test_break_continue(self):
code = makecode("""
|z="test"
|for x in [1,2,3,4]:
| print(x)
| y=2
| break
| z=2
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
y = module_scope.resolve('y')
self.assertIsNum(y)
z = module_scope.resolve('z')
self.assertIsStr(z)
def test_inc_dec(self):
code = makecode("""
|class A:
| def __init__(self):
| self.x = 4
|x=1
|z=2
|x+=z
|a=A()
|a.x+=1
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
x = module_scope.resolve('x')
print("x" + str(x))
a = module_scope.resolve('a')
self.assertIsNum(a.scope['x'])
print("a.x: " + str(a.scope['x']))
def test_complex_example(self):
code = makecode("""
|class EmptyWareHouseException:
| pass
|class FullWareHouseException(Exception):
| pass
|class Farm:
| def __init__(self,name,size):
| self.name = name
| self.sizeOfWareHouse = size
| self.currentSize = 0
| def produceOneUnit(self):
| if self.currentSize==self.sizeOfWareHouse:
| raise FullWareHouseException()
| self.currentSize += 1
| def consumeOneUnit(self):
| if self.currentSize==0:
| raise EmptyWareHouseException()
| self.currentSize -= 1
|
|farm = Farm(size=50,name='Farm from wonderland')
|farm.produceOneUnit()
|farm.consumeOneUnit()
|farm.produceOneUnit()
""")
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
print("P:" + str(self.finalParser.problems))
print("W:" + str(self.finalParser.warnings))
#self.assertTrue(self.finalParser.nondet)
#self.assertEqual(len(self.finalParser.problems), 2)
#self.assertEqual(len(self.finalParser.warnings), 0)
#problem_symbols = {problem.symbol for problem in self.finalParser.problems}
#self.assertEqual(problem_symbols, {'p','q'})
def test_complex_example2(self):
code = makecode('''
|class Element(object):
| def __init__(self):
| self.data = 10
| def printX(self):
| print(self.x) #error
|
|instanceOfElement = Element()
|testList = [instanceOfElement, "test"]
|testList[0].printX()
|10 + testList[1] #error
''')
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.finalParser.eval_in_root(node)
print("P:" + str(self.finalParser.problems))
print("W:" + str(self.finalParser.warnings))
def test_tuple_assign(self):
code = makecode('''
|def makePair(x,y):
| return (x,y)
|a,b = makePair(5,'five')
''')
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
def test_help(self):
code = makecode('''
|def func():
| """
| vnjvndfkvns cnscksdcnsj
| """
| x=4
|i[3]
''')
node = ast.parse(code, mode='exec')
print(utils.astNode_to_tree(node))
self.parser.eval_in_root(node)
module_scope = self.parser.root_scope
for problem in self.parser.problems:
print(str(problem))
f = module_scope.resolve('func')
if hasattr(f, 'docstring'):
print(f.docstring)
def get_auto_completion(host, client_address, buffer_used, variable_line, line_number):
'''
This method is called when user presses '.' symbol.
We want to find all possible attributes for given symbol on given buffer at given line.
Call eval_in_root plus send nodeAst - means evaluation will stop once given node is reached.
'''
try:
#Get leading spaces so the indent matches current level
variable_line = variable_line.replace('\t', ' ')
leading_spaces = len(variable_line) - len(variable_line.lstrip())
# Get only relevant part of line
line_modified = utils.getObjectStringFromLine(variable_line)
# Replace given line with string which will be resolved - much easier for our solution
resolving_name = 'target_for_completion'
buffer = buffer_used
buffer[line_number - 1] = variable_line[0:leading_spaces] + resolving_name + ' = ' + line_modified
buffer = buffer[0:line_number]
# Parse modified buffer and eval
data = buffer_to_string(buffer)
lexer = python_lexer.PythonLexer()
lexer.input(data)
lexer = python_lexer.PythonLexer()
res = python_parser.parse_data(data,lexer)
tree, del_parts = utils.traverse_ast_test(res)
ast_tree = utils.parse_with_ast(res)
# print("TREE:"+str(utils.astNode_to_tree(ast_tree)))
del_lines = []
for delPart in del_parts:
for i in range(delPart[0],delPart[1]+1):
del_lines.append(i)
del_lines+=res.emptyLinesNums
temp={line for line in del_lines}
del_lines = [line for line in temp]
del_lines.sort()
current_line_number = utils.getCurrentLineNum(line_number, del_lines)
parser = FinalParser(1)
parser.eval_in_root(ast_tree)
# print(str(parser.scopes[0]))
#Remove inf_ attribtues since those are used for internal purposes
list_of_all = parser.get_all_possible_attr(resolving_name)
reduced_list = []
for item in list_of_all:
if not item[0].startswith('inf_'):
reduced_list.append(item)
# Respond to the client.
response_completion = json.dumps({'options' : reduced_list})
host.respond(bytes(response_completion, 'UTF-8'), client_address)
except:
traceback.print_exc()
# Send an empty list if any error occurred
list_for_completion = []
response_completion_error = json.dumps({'options' : list_for_completion})
host.respond(bytes(response_completion_error, 'UTF-8'), client_address)
def parse_and_validate(host, dictionaryID, client_address, number_of_iterations):
"""
Main method which evaluates whole code and sends respond with errors and warnings.
"""
try:
start_time = time.time() * 1000
log_to_file("START OF VALIDATION: "+str(start_time)+", Number of iterations: "+str(number_of_iterations))
buffer = openBuffers[dictionaryID]
problems_list = []
warnings_list = []
# Decode the data
data = buffer_to_string(buffer)
lexer = python_lexer.PythonLexer()
lexer.input(data)
#for token in lexer:
# print(token.value)
lexer = python_lexer.PythonLexer()
res = python_parser.parse_data(data,lexer)
#log_time("AFTER PARSE DATA: ", time.time() * 1000, start_time)
tree, del_parts = utils.traverse_ast_test(res)
#log_time("AFTER TRAVERSE AST: ", time.time() * 1000, start_time)
ast_tree = utils.parse_with_ast(res)
#log_time("AFTER PARSE WITH AST: ", time.time() * 1000, start_time)
parser=FinalParser(number_of_iterations)
parser.eval_in_root(ast_tree)
del_lines = []
for delPart in del_parts:
for i in range(delPart[0],delPart[1]+1):
del_lines.append(i)
#log_time("AFTER EVAL IN ROOT: ", time.time() * 1000, start_time)
#processing syntax problems
for line in del_lines:
p = []
p.append(line)
p.append('Invalid syntax.')
problems_list.append(p)
del_lines+=res.emptyLinesNums
temp={line for line in del_lines}
del_lines = [line for line in temp]
del_lines.sort()
list_of_used_lines = []
#Problems
for problem in parser.problems:
if not hasattr(problem.node, 'processed'):
problem.node.lineno=utils.getOriginLineNum(problem.node.lineno,del_lines)
problem.node.processed=1
if not (problem.node.lineno in list_of_used_lines):
b = []
b.append(problem.node.lineno)
b.append(str(problem))
problems_list.append(b)
list_of_used_lines.append(problem.node.lineno)
#Warnings
for warning in parser.warnings:
if not hasattr(warning.node, 'processed'):
warning.node.lineno=utils.getOriginLineNum(warning.node.lineno,del_lines)
warning.node.processed=1
w = []
w.append(warning.node.lineno)
w.append(str(warning))
warnings_list.append(w)
log_to_file("Problems: "+str(problems_list))
log_to_file("Warnings: "+str(warnings_list))
log_to_file("Validation completed...")
problems = json.dumps({'problems' : problems_list, 'warnings' : warnings_list})
#print("DUMPED THING: "+str(problems))
host.respond(bytes(problems, "UTF-8"), client_address)
host.validationRunning = False
log_to_file('----------------------------')
except IndentationError as error:
log_to_file("Indentation error in parsing.")
traceback.print_exc()
indent_error = json.dumps({'message' : error.msg, 'line_number' : error.lineno})
host.respond(bytes(indent_error,"UTF-8"), client_address)
host.validationRunning = False
except python_parser.RobustParserError as error:
log_to_file("Error in parsing: returning correct line number.")
b = []
b.append(error.data.lineno)
b.append("invalid syntax")
problems_list.append(b)
problems = json.dumps({'problems' : problems_list, 'warnings' : warnings_list})
host.respond(bytes(problems, "UTF-8"), client_address)
host.validationRunning = False
except Exception as error:
log_to_file("Error in parsing: ")
traceback.print_exc()
#connection.sendall(bytes("endOfValidation: "+error, "utf-8"))
#host.respond(bytes("endOfValidation", "utf-8"), client_address)
error_problems_response = json.dumps({'problems' : [], 'warnings' : []})
host.respond(bytes(error_problems_response, "UTF-8"), client_address)
host.validationRunning = False
try:
if len(sys.argv) == 2:
path = sys.argv[1]
elif len(sys.argv) == 3:
path = sys.argv[1]
count_iterations = sys.argv[2]
else:
raise InvalidArgsException()
f = open(path,'r')
data = f.read()
abs_syn_tree = ast.parse(data)
if count_iterations is None:
final_parser = FinalParser()
else:
final_parser = FInalParser(count_iterations)
final_parser.eval_in_root(abs_syn_tree)
if len(final_parser.problems)==0 and len(final_parser.warnings)==0:
print('No errors were detected')
else:
print('Following errors were detected:')
rows = []
for p in final_parser.problems:
print(pretty_str(p))
for w in final_parser.warnings:
print(pretty_str(w,'W'))
except SyntaxError as e:
