def test_lexical(self):
# sentence = "if obj.get(x)[1] < a[1]"
# x = parser.lexical_analyze(sentence)
#
# sentence = "if obj.get(x).get(y) < a[1]"
# x = parser.lexical_analyze(sentence)
# print(x)
sentence = "x-> y"
x = parser.lexical_analyze(sentence)
print(x)
def evaluate(expression):
'''
:param expression: the expression argument is parsed and evaluated.
:return: the result of the evaluated expression
Similar to python eval function. The evaluate function only compile a single expression and returns the value of the expression.
Syntax errors are reported as exceptions.
Todo: in this version, we keep using python eval(). Will change in next version.
'''
tokenList = lexical_analyze(expression)
#Add Array or Dict to array definition or dict definition
#E.g. [1,[1,2]] => Array([1,Array[1,2]])
index = 0
token = ''
modifiedExp = ''
bracket_stack = [] #stack of '[', ']'
brace_stack = [] #stack of '{', '}'
while index < len(tokenList):
token = tokenList[index][1]
#avoid case like x[index] and [x[1], 2, 3]
#1. '[' is part of array definition
if token == '[' and (index == 0 or
(index > 0
and tokenList[index - 1][0] not in VARRANGE
and tokenList[index - 1][1] not in [']', ')'])):
modifiedExp += "Array(["
#2. '[' is used for index
elif token == '[':
modifiedExp += token
bracket_stack.append('[')
elif token == ']':
#3. ']' is part of index visit
if bracket_stack:
bracket_stack.pop()
modifiedExp += token
#4. ']' is part of array definition
else:
modifiedExp += "])"
#5. '{}' is dictionary definition
elif token == '{':
modifiedExp += "Dict({"
elif token == '}':
modifiedExp += "})"
#6. other cases
else:
modifiedExp += token
modifiedExp += ' '
index += 1
expression = modifiedExp
#TODO: be careful about variable scope, to be tested later
#step1: store all variable in global stack to a dictionary, start from bottom of stack.
from module.basemodule import basemodule
varDict = {}
for variablePair in glb.variable_stack:
if not isinstance(variablePair, basemodule):
varDict[variablePair[0]] = variablePair[1].var_obj
#step2: use python eval
result = eval(expression, glb.function_map, varDict)
return result
def execute(statement):
'''
:param statement: statement to be executed
:return the value returned by the statement if any, otherwise return None
New variables will be added to variable stack.
Expression type: 1) variable definition 2) if, for, while statement's conditions 3) pure expression
**********
For right operand, we could invoke evaluate function directly.
For left operand, we could invoke evaluate function for the parameters and indices.
However we can not use evaluate function for the entire left operand because if the returned object of left operand is a string type,
we can not change the object value by simply assign the right operand value to it.
Eg. x = 4
left = evaluate(x)
right = evaluate(4)
left = right
x is not changed!
'''
tokenList = lexical_analyze(statement)
#statement is an assignment. Left operand must be a variable
#Three types of left operand:
#1. object -> Need to store new variable to stack
#2. object.function()
#3. object[index/key] -> Need to record the index or key
#if it is an assignment
if equalToken in tokenList:
indexOfEqual = tokenList.index(equalToken)
leftOpTokens = tokenList[:indexOfEqual]
rightOpTokens = tokenList[indexOfEqual + 1:]
#get string expression of right operand
rightOpStr = ''
for token in rightOpTokens:
rightOpStr += token[1] + ' '
#the object of the left operand
leftObjectName = leftOpTokens[0][1]
#variable is not defined yet
if not containVariableInGlbStack(leftObjectName):
#second or third type of left operand
if len(leftOpTokens) > 1:
raise Exception("\"" + leftObjectName +
"\" is undefined in statement: " + statement)
#first type of left operand
else:
rightOpValue = evaluate(rightOpStr)
variableObj = variable(leftObjectName, rightOpValue)
glb.variable_stack.append([leftObjectName, variableObj])
#variable is defined previously
else:
variableInstance = getVariableFromGlbStack(
leftObjectName) #get variable from global stack
rightOpValue = evaluate(
rightOpStr) #evaluate the right operand and get the value
variableInstance.var_flag = glb.flag_dict['changed']
#single variable, store in the global stack directly
if len(leftOpTokens) == 1:
variableInstance.var_obj = rightOpValue
else:
#analyze token before ".", find indices visited. Format: obj[index/key]...[index/key]
leftOpObject = variableInstance.var_obj
storeVisitedIndex(leftOpTokens)
#compile until the last "." or "[". To split the left operand into 2 parts, the first part is non-primitive and second part is primitive part
#find the index of last '.' OR '['
index = 0
splitIndex = 0
while index < len(leftOpTokens):
if leftOpTokens[index][1] == '.' or leftOpTokens[index][
1] == '[':
splitIndex = index
index += 1
index = 1 #reset index
exp = ''
token = '' #single token
currentObject = leftOpObject #object compiled until now
while index < len(leftOpTokens):
token = leftOpTokens[index][1]
#attribute or function call
if token == '.':
index += 1
token = leftOpTokens[index][1]
if token not in dir(currentObject):
raise Exception("AttributeError: \'" +
leftObjectName +
"\' object has no attribute \'" +
token + "\'")
else:
if index >= splitIndex:
setattr(currentObject, token, rightOpValue)
else:
currentObject = getattr(currentObject, token)
#function call
elif token == '(':
bracket_stack = []
bracket_stack.append(token)
isMultipleParam = False
while index < len(leftOpTokens) - 1 and bracket_stack:
index += 1
token = leftOpTokens[index][1]
#check if it is multiple parameters
if token == ',' and len(bracket_stack) == 1:
isMultipleParam = True
if token == '(':
bracket_stack.append(token)
elif token == ')':
bracket_stack.pop()
if bracket_stack:
exp += token
paramResult = evaluate(exp)
if isMultipleParam:
currentObject = currentObject(
*paramResult
) #execute the function, paramResult may be a tuple with multiple parameters
else:
currentObject = currentObject(paramResult)
#indices or keys
elif token == '[':
bracket_stack = []
bracket_stack.append(token)
while index < len(leftOpTokens) - 1 and bracket_stack:
index += 1
token = leftOpTokens[index][1]
if token == '[':
bracket_stack.append(token)
elif token == ']':
bracket_stack.pop()
if bracket_stack:
exp += token
paramResult = evaluate(exp)
if index >= splitIndex:
currentObject[paramResult] = rightOpValue
else:
currentObject = currentObject[
paramResult] #execute the function
else:
raise Exception("Invalid syntax at " + token +
" in statement " + statement)
token = ''
exp = ''
index += 1
storePointer(leftOpTokens, rightOpTokens)
else:
evaluate(statement)
def test_parse(self):
#Test1: function
sentence = "function sort(array)"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
self.assertEquals(gram, 'function_def')
self.assertEquals(tokenList, ((12, 'function'), (1, 'sort'), (54, '('), (0, 'array'), (55, ')')))
self.assertEquals(param_list, ['sort', ('array',)])
sentence = "function sort()"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
print(len(param_list[1]))
#Test2: variable definition in TYPE VAR = XXX format
sentence = "Stack x = [1,2]"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
self.assertEquals(gram, 'variable_def')
self.assertEquals(exeToken, 'x = Stack([ 1 , 2 ] )')
self.assertEquals(tokenList, ((1, 'Stack'), (0, 'x'), (51, '='), (66, '['), (2, '1'), (64, ','), (2, '2'), (67, ']')))
self.assertEquals(param_list, ['x'])
#Test3: variable definition expression
sentence = "x = 'fang'"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
self.assertEquals(gram, 'expression')
self.assertEquals(exeToken, "x = 'fang' ")
self.assertEquals(tokenList, ((0, 'x'), (51, '='), (2, "'fang'")))
self.assertEquals(param_list, [])
#Test4: statements:for loop
sentence = "for x in range(4)"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
self.assertEquals(gram, 'statement')
self.assertEquals(exeToken, "")
self.assertEquals(tokenList, ((5, 'for'), (0, 'x'), (14, 'in'), (1, 'range'), (54, '('), (2, '4'), (55, ')')))
# execute function not finished yet
# self.assertEquals(param_list, ['x', 0])
sentence = "for x = 0 to 9"
# gram, tokenList, exeToken, param_list = parser.parse(sentence)
# self.assertEquals(gram, 'statement')
# self.assertEquals(exeToken, "")
# self.assertEquals(tokenList, ((5, 'for'), (0, 'x'), (51, '='), (2, '0'), (18, 'to'), (2, '9')))
# #execute function not finished yet
# self.assertEquals(param_list, ['x', range(0, 0)])
#
sentence = "for x = 0 to 9 step 2"
print(parser.lexical_analyze(sentence))
# # gram, tokenList, exeToken, param_list = parser.parse(sentence)
# # self.assertEquals(gram, 'statement')
# # self.assertEquals(exeToken, "")
# # self.assertEquals(tokenList, ((5, 'for'), (0, 'x'), (51, '='), (2, '0'), (18, 'to'), (2, '9'), (19, 'step'), (2, '2')))
# #execute function not finished yet
# # self.assertEquals(param_list, ['x', range(0, 9, 2)])
#
#
sentence = "while x < 9"
gram, tokenList, exeToken, param_list = parser.parse(sentence)
self.assertEquals(gram, 'statement')
self.assertEquals(exeToken, "x < 9 ")
self.assertEquals(tokenList, ((6, 'while'), (0, 'x'), (45, '<'), (2, '9')))
#execute function not finished yet
self.assertEquals(param_list, ["x < 9 "])
def evaluate(expression):
"""
:param expression: the expression argument is parsed and evaluated.
:return: the result of the evaluated expression
Similar to python eval function. The evaluate function only compile a single expression and returns the value of the expression.
Syntax errors are reported as exceptions.
Todo: in this version, we keep using python eval(). Will change in next version.
"""
tokenList = lexical_analyze(expression)
# Add Array or Dict to array definition or dict definition
# E.g. [1,[1,2]] => Array([1,Array[1,2]])
index = 0
token = ""
modifiedExp = ""
bracket_stack = [] # stack of '[', ']'
brace_stack = [] # stack of '{', '}'
while index < len(tokenList):
token = tokenList[index][1]
# avoid case like x[index] and [x[1], 2, 3]
# 1. '[' is part of array definition
if token == "[" and (
index == 0
or (index > 0 and tokenList[index - 1][0] not in VARRANGE and tokenList[index - 1][1] not in ["]", ")"])
):
modifiedExp += "Array(["
# 2. '[' is used for index
elif token == "[":
modifiedExp += token
bracket_stack.append("[")
elif token == "]":
# 3. ']' is part of index visit
if bracket_stack:
bracket_stack.pop()
modifiedExp += token
# 4. ']' is part of array definition
else:
modifiedExp += "])"
# 5. '{}' is dictionary definition
elif token == "{":
modifiedExp += "Dict({"
elif token == "}":
modifiedExp += "})"
# 6. other cases
else:
modifiedExp += token
modifiedExp += " "
index += 1
expression = modifiedExp
# TODO: be careful about variable scope, to be tested later
# step1: store all variable in global stack to a dictionary, start from bottom of stack.
from module.basemodule import basemodule
varDict = {}
for variablePair in glb.variable_stack:
if not isinstance(variablePair, basemodule):
varDict[variablePair[0]] = variablePair[1].var_obj
# step2: use python eval
result = eval(expression, glb.function_map, varDict)
return result
def execute(statement):
"""
:param statement: statement to be executed
:return the value returned by the statement if any, otherwise return None
New variables will be added to variable stack.
Expression type: 1) variable definition 2) if, for, while statement's conditions 3) pure expression
**********
For right operand, we could invoke evaluate function directly.
For left operand, we could invoke evaluate function for the parameters and indices.
However we can not use evaluate function for the entire left operand because if the returned object of left operand is a string type,
we can not change the object value by simply assign the right operand value to it.
Eg. x = 4
left = evaluate(x)
right = evaluate(4)
left = right
x is not changed!
"""
tokenList = lexical_analyze(statement)
# statement is an assignment. Left operand must be a variable
# Three types of left operand:
# 1. object -> Need to store new variable to stack
# 2. object.function()
# 3. object[index/key] -> Need to record the index or key
# if it is an assignment
if equalToken in tokenList:
indexOfEqual = tokenList.index(equalToken)
leftOpTokens = tokenList[:indexOfEqual]
rightOpTokens = tokenList[indexOfEqual + 1 :]
# get string expression of right operand
rightOpStr = ""
for token in rightOpTokens:
rightOpStr += token[1] + " "
# the object of the left operand
leftObjectName = leftOpTokens[0][1]
# variable is not defined yet
if not containVariableInGlbStack(leftObjectName):
# second or third type of left operand
if len(leftOpTokens) > 1:
raise Exception('"' + leftObjectName + '" is undefined in statement: ' + statement)
# first type of left operand
else:
rightOpValue = evaluate(rightOpStr)
variableObj = variable(leftObjectName, rightOpValue)
glb.variable_stack.append([leftObjectName, variableObj])
# variable is defined previously
else:
variableInstance = getVariableFromGlbStack(leftObjectName) # get variable from global stack
rightOpValue = evaluate(rightOpStr) # evaluate the right operand and get the value
variableInstance.var_flag = glb.flag_dict["changed"]
# single variable, store in the global stack directly
if len(leftOpTokens) == 1:
variableInstance.var_obj = rightOpValue
else:
# analyze token before ".", find indices visited. Format: obj[index/key]...[index/key]
leftOpObject = variableInstance.var_obj
storeVisitedIndex(leftOpTokens)
# compile until the last "." or "[". To split the left operand into 2 parts, the first part is non-primitive and second part is primitive part
# find the index of last '.' OR '['
index = 0
splitIndex = 0
while index < len(leftOpTokens):
if leftOpTokens[index][1] == "." or leftOpTokens[index][1] == "[":
splitIndex = index
index += 1
index = 1 # reset index
exp = ""
token = "" # single token
currentObject = leftOpObject # object compiled until now
while index < len(leftOpTokens):
token = leftOpTokens[index][1]
# attribute or function call
if token == ".":
index += 1
token = leftOpTokens[index][1]
if token not in dir(currentObject):
raise Exception(
"AttributeError: '" + leftObjectName + "' object has no attribute '" + token + "'"
)
else:
if index >= splitIndex:
setattr(currentObject, token, rightOpValue)
else:
currentObject = getattr(currentObject, token)
# function call
elif token == "(":
bracket_stack = []
bracket_stack.append(token)
isMultipleParam = False
while index < len(leftOpTokens) - 1 and bracket_stack:
index += 1
token = leftOpTokens[index][1]
# check if it is multiple parameters
if token == "," and len(bracket_stack) == 1:
isMultipleParam = True
if token == "(":
bracket_stack.append(token)
elif token == ")":
bracket_stack.pop()
if bracket_stack:
exp += token
paramResult = evaluate(exp)
if isMultipleParam:
currentObject = currentObject(
*paramResult
) # execute the function, paramResult may be a tuple with multiple parameters
else:
currentObject = currentObject(paramResult)
# indices or keys
elif token == "[":
bracket_stack = []
bracket_stack.append(token)
while index < len(leftOpTokens) - 1 and bracket_stack:
index += 1
token = leftOpTokens[index][1]
if token == "[":
bracket_stack.append(token)
elif token == "]":
bracket_stack.pop()
if bracket_stack:
exp += token
paramResult = evaluate(exp)
if index >= splitIndex:
currentObject[paramResult] = rightOpValue
else:
currentObject = currentObject[paramResult] # execute the function
else:
raise Exception("Invalid syntax at " + token + " in statement " + statement)
token = ""
exp = ""
index += 1
storePointer(leftOpTokens, rightOpTokens)
else:
evaluate(statement)