def insertToString(self, cls, basecls, visitor):
# visitor.logger.debug('insertToString', cls, basecls)
name = 'toString'
if name not in cls.symbols:
# visitor.logger.debug('insertToString do', cls, basecls)
clsname = ast.Call(ast.Identifier('getClassName'), [])
thisptr = ast.Call(ast.Identifier('formatId'), [])
expr = None
if 'name' in cls.symbols:
# visitor.logger.debug('insertToString with name', cls, basecls, clsname, thisptr)
expr = ast.BinaryOp(
'%', ast.makeLiteral('%s(name=%s,id=%s)'),
ast.TupleLiteral(
[clsname, ast.Identifier('name'), thisptr]))
else:
# visitor.logger.debug('insertToString without name', cls, basecls, clsname, thisptr)
expr = ast.BinaryOp('%', ast.makeLiteral('%s(id=%s)'),
ast.TupleLiteral([clsname, thisptr]))
func = ast.FuncDef([], name,
ast.FuncSpec([],
ast.makePrimitiveType('string')),
ast.StatementBody([ast.Return(expr)]))
# visitor.logger.debug('insertFunc ok', name, cls, basecls, cls.primaryVars, func)
cls.definitions.append(func)
func.setOwner(cls)
visitor.visitNewItem(func)
def parseFunctionParameters(self): identifiers = [] if self.peekTokenIs(token.RPAREN): self.nextToken() return identifiers self.nextToken() ident = ast.Identifier() ident.Token = self.curToken ident.Value = self.curToken.Literal identifiers.append(ident) while self.peekTokenIs(token.COMMA): self.nextToken() self.nextToken() ident = ast.Identifier() ident.Token = self.curToken ident.Value = self.curToken.Literal identifiers.append(ident) if not self.expectPeek(token.RPAREN): return None return identifiers
def parse_method_call(self, left):
expr = ast.MethodCall(self.cur_tok, left, [])
while self.peek_in(self.arg_tokens) or self.peek_in(
token.keywords.values()):
self.next()
if self.cur_in(token.keywords.values()):
expr.pattern.append(ast.Identifier(self.cur_tok))
continue
arg = lambda val: ast.Argument(self.cur_tok, val)
handlers = {
token.ID: lambda: ast.Identifier(self.cur_tok),
token.LPAREN: lambda: arg(self.parse_grouped_expr()),
token.NUM: lambda: arg(self.parse_num()),
token.NULL: lambda: arg(self.parse_null()),
token.TRUE: lambda: arg(self.parse_bool()),
token.FALSE: lambda: arg(self.parse_bool()),
token.STR: lambda: arg(self.parse_string()),
token.PARAM: lambda: arg(ast.Identifier(self.cur_tok)),
token.LSQUARE: lambda: arg(self.parse_array_or_map()),
token.LBRACE: lambda: arg(self.parse_block_literal()),
}
handler = handlers[self.cur_tok.type]
expr.pattern.append(handler())
if len(expr.pattern) == 0:
self.err("expected at least one item in a pattern")
return None
return expr
def generateSimpleFieldCall(self, cls, basecls, basefunc, field,
invokeFuncName):
callinfo = ast.Call(
ast.AttrRef(ast.Identifier(field.name), invokeFuncName),
[ast.Identifier(param.name) for param in basefunc.spec.params])
# teststmts = [ast.CallStatement(ast.Call(ast.Identifier('println'), [ast.makeLiteral('GenerateTreeFunc check ' + basefunc.name + ' ' + field.name),
# ast.Identifier(field.name),
# ast.Call(ast.Identifier('toString'), [])]))]
teststmts = []
isStatement = basefunc.spec.returnType is None or (
isinstance(basefunc.spec.returnType, ast.UserType)
and basefunc.spec.returnType.fullpath == 'void')
if isStatement:
body = ast.StatementBody([
# ast.Call(ast.Identifier('println'), [ast.makeLiteral('GenerateTreeFunc ' + basefunc.name + ' ' + field.name),
# ast.Call(ast.AttrRef(ast.Identifier(field.name), 'toString'), []),
# ast.Call(ast.Identifier('toString'), [])]),
callinfo
])
branch = ast.IfBranch(
ast.BinaryOp('!=', ast.Identifier(field.name), ast.Nil()),
body)
return ast.StatementBody(teststmts +
[ast.IfStatement([branch], None)])
return ast.IfElseExpr(
ast.BinaryOp('!=', ast.Identifier(field.name), ast.Nil()),
callinfo, ast.Nil())
def TestString(self):
program = ast.Program()
program.Statements = [
ast.LetStatement(
tkn.Token(tkn.LET, "let"),
ast.Identifier(tkn.Token(tkn.IDENT, "myVar"), "myVar"),
ast.Identifier(tkn.Token(tkn.IDENT, "anotherVar"),
"anotherVar"))
]
if program.String() != "let myVar = anotherVar":
raise Exception(f'program.String() wrong got={program.String()}')
def TestString():
program = ast.Program(Statements=[
ast.LetStatement(
Token=_token.Token(Type=_token.tokens.LET, Literal="let"),
Name=ast.Identifier(Token=_token.Token(Type=_token.tokens.IDENT,
Literal="myVar"),
Value="myVar"),
Value=ast.Identifier(Token=_token.Token(Type=_token.tokens.IDENT,
Literal="anotherVar"),
Value="anotherVar"))
])
if program.String() == "let myVar = anotherVar;":
print("TestString test is success")
def _expr_component(self):
"""Parse an expr component (components can be separated by an operator).
"""
curtok = self.cur_token
if self.cur_token.type == 'INT':
self._get_next_token()
return ast.IntConstant(curtok.val)
elif self.cur_token.type in ('FALSE', 'TRUE'):
self._get_next_token()
return ast.BoolConstant(curtok.val)
elif self.cur_token.type == 'ID':
self._get_next_token()
if self.cur_token.type == '(':
# ID followed by '(' is function application
return self._app(curtok.val)
else:
return ast.Identifier(curtok.val)
elif self.cur_token.type == '(':
self._get_next_token()
expr = self._expr()
self._match(')')
return expr
elif self.cur_token.type == 'IF':
return self._ifexpr()
elif self.cur_token.type == 'LAMBDA':
return self._lambda()
else:
self._error("Don't support {} yet".format(curtok.type))
def convert_string_literal(text):
# process string evaluation
# print('convert_string_literal', text)
pos = text.find('$')
if pos >= 0:
# check if it's preceded by a backslash
if pos == 0 or text[pos-1] != '\\':
if pos + 1 < len(text):
nexttag = text[pos+1]
if nexttag == '{':
endpos = text.find('}', pos+1)
if endpos >= 0:
exprstr = text[pos+2:endpos]
expr = parser.exprParser.parse(exprstr)
newtext = text[:pos] + "%s" + text[endpos+1:]
# print('convert_string_literal brace part', text, exprstr, expr, newtext)
realtext, exprs = convert_string_literal(newtext)
return realtext, [expr] + exprs
elif nexttag == '_' or nexttag.isalpha():
idname = nexttag
i = pos + 2
while i < len(text):
ch = text[i]
if ch == '_' or ch.isalpha():
idname += ch
else:
break
# print('convert_string_literal idname', idname, text)
expr = ast.Identifier(idname)
newtext = text[:pos] + "%s" + text[i:]
realtext, exprs = convert_string_literal(newtext)
return realtext, [expr] + exprs
return text, []
def patternMatching():
global _functions
global _args
global _pattern_functions
for func in _functions:
if len(_functions[func]) == 1:
_functions[func] = _functions[func][func]
_args[func] = _args[func][func]
else:
_pattern_functions[func] = True
tree = _functions[func][func]
argsList = _args[func][func]
for patternKey in _functions[func]:
if patternKey != func:
argsValues = _args[func][patternKey]
eqs = []
for (arg, value) in zip(argsList, argsValues):
if arg != value:
eqs += [
ast.Binop(ast.Identifier(arg), "==", value)
]
cond = eqs[0]
for i in range(1, len(eqs)):
cond = ast.Binop(cond, "and", eqs[i])
tree = ast.Conditional(cond, _functions[func][patternKey],
tree)
_functions[func] = tree
_args[func] = argsList
def _expr_component(self):
curtok = self.cur_token
if self.cur_token.type == "INT":
self._get_next_token()
return ast.IntConstant(curtok.val)
elif self.cur_token.type in ("FALSE", "TRUE"):
self._get_next_token()
return ast.BoolConstant(curtok.val)
elif self.cur_token.type == "ID":
self._get_next_token()
if self.cur_token.type == "(":
return self._app(curtok.val)
else:
return ast.Identifier(curtok.val)
elif self.cur_token.type == "(":
self._get_next_token()
expr = self._expr()
self._match(")")
return expr
elif self.cur_token.type == "IF":
return self._ifexpr()
elif self.cur_token.type == "LAMBDA":
return self._lambda()
else:
self._error(f"Don't suppor {curtok.type} yet.")
def p_lambda_expression(t):
'''lambda : LAMBDA argList ARROW expression'''
_, _, args, _, expression = t
global lambdaCounter
funcName = "lambda {}".format(lambdaCounter)
lambdaCounter += 1
global _lambda_childrens
global _lambda_closure
for fName in _lambda_childrens:
if fName not in _lambda_closure:
_lambda_closure[fName] = []
_lambda_closure[fName] += args
_lambda_childrens += [funcName]
wheres = []
name = ""
primal = True
for arg in args:
tp, vl, check = getName(arg)
name += "(" + tp + ")" + vl
if not check:
primal = False
if primal:
name = funcName
global _names
global _names_aux
_names[funcName] = _names_aux
_names_aux = set()
global _dependence
global _dependence_aux
_dependence[funcName] = list(_dependence_aux)
_dependence_aux = set()
global _functions
if funcName in _functions:
_functions[funcName][name] = expression
else:
_functions[funcName] = {name: expression}
global _args
if funcName in _args:
_args[funcName][name] = args
else:
_args[funcName] = {name: args}
global _whereDict
_whereDict[funcName] = wheres
if funcName != 'main' and type(
expression).__name__ == "Application" and wheres == []:
global _eta_list
_eta_list += [funcName]
t[0] = ast.Identifier(funcName)
def parseLetStatement(self):
print("parseLetStatement(): ", self.curToken)
stmt = ast.LetStatement()
stmt.Token = self.curToken
print("parseLetStatement(): ", stmt)
stmt.printout()
if not self.expectPeek(token.IDENT):
return None
stmt.Name = ast.Identifier()
stmt.Name.Token = self.curToken
stmt.Name.Value = self.curToken.Literal
stmt.printout()
if not self.expectPeek(token.ASSIGN):
return None
"""
while not self.curTokenIs(token.SEMICOLON):
self.nextToken()
"""
self.nextToken()
stmt.Value = self.parseExpression(LOWEST)
if self.peekTokenIs(token.SEMICOLON):
self.nextToken()
return stmt
def generateCloneStatements(self, cls, basecls, visitor, stmts):
# visitor.logger.debug('generateCloneStatements', visitor, basecls)
assert len(stmts) <= len(cls.primaryVars)
newfields = [
ast.NamedExpressionItem(cls.primaryVars[i].name, stmts[i])
for i in range(len(cls.primaryVars))
]
return [ast.Return(ast.Call(ast.Identifier(cls.name), [], newfields))]
def generateInitializeOwnerCall(self, cls, basecls, basefunc, field):
# print('generateInitializeOwnerCall', basefunc.name, cls, field, basefunc)
# msgstmt = ast.Call(ast.Identifier('println'), [ast.makeLiteral('generate initializeOwner start ' + field.name),
# ast.Identifier(field.name),
# ast.Call(ast.Identifier('toString'), [])])
body = ast.StatementBody([
# ast.Call(ast.Identifier('println'), [ast.makeLiteral('generate initializeOwner ' + field.name),
# ast.Identifier(field.name),
# ast.Call(ast.Identifier('toString'), [])]),
ast.Call(ast.AttrRef(ast.Identifier(field.name), 'setOwner'),
[ast.This()]),
self.generateSimpleFieldCall(cls, basecls, basefunc, field,
'initializeOwner')
])
branch = ast.IfBranch(
ast.BinaryOp('!=', ast.Identifier(field.name), ast.Nil()), body)
# return ast.StatementBody([msgstmt, ast.IfStatement([branch], None)])
return ast.IfStatement([branch], None)
def vAssignmentStatement(self, node):
if isinstance(
node.expr,
ast.Constructor) and node.expr.type.resolveType().isVector():
self.w('New')
self.w('(')
self.visit(node.lval)
cons = node.expr
while isinstance(cons, ast.Constructor):
self.w(',')
#print cons.params
self.visit(cons.params.children[0])
if len(cons.params.children) == 1:
cons = None
break
else:
cons = cons.params.children[1]
if cons != None:
print '****************Need to initialize******************'
self.w(')')
elif isinstance(node.expr, ast.AssignmentStatement):
self.visit(node.expr)
self.p(';')
tmp = node.expr
if isinstance(node.expr.lval, str):
node.expr = ast.Identifier(node.expr.lval)
else:
node.expr = node.expr.lval
node.expr.symtab = node.symtab
self.visit(node)
node.expr = tmp
else:
ident = node.lval
ident.symtab = node.symtab
oldexpr = node.expr
if node.operator != '=':
node.expr = ast.BinaryOp(
ident, self.assignmentOperators[node.operator],
ast.Parenthesis(node.expr))
node.expr.symtab = node.symtab
tpe_ident = ident.resolveType().type
tpe_expr = node.expr.resolveType().type
#if isinstance(ident.resolveType(), ast.StructType):
#print ident.resolveType().id
#print tpe_ident
#if isinstance(node.expr.resolveType().resolveType(), ast.StructType):
#print node.expr.resolveType().resolveType().id
#print tpe_expr
if tpe_ident != tpe_expr and not (
(tpe_ident == 'float' and tpe_expr == 'double') or
(tpe_ident == 'double' and tpe_expr == 'float')):
node.expr = ast.CastExpression(ast.Type(tpe_ident), node.expr)
node.expr.symtab = node.symtab
self.visit(node.lval)
self.w(':=')
self.visit(node.expr)
node.expr = oldexpr
def addMixinFunc(self, visitor, script, f, owner, mixincls):
# visitor.logger.debug('addMixinFunc', f.name, owner, f, mixincls)
if not owner.hasSymbol(f.name):
callinfo = ast.Call(ast.AttrRef(script.args[1].clone(), f.name), [ast.Identifier(param.name) for param in f.spec.params])
# visitor.logger.debug('addMixinFunc callinfo', f.name, owner, f, mixincls, callinfo, callinfo.caller)
newfunc = ast.FuncDef([], f.name, f.spec.clone(), ast.StatementBody([ast.Return(callinfo)]))
owner.definitions.append(newfunc)
newfunc.setOwner(owner)
visitor.visitNewItem(newfunc)
def _parse_function_parameters(self) -> Optional[List[ast.Identifier]]:
identifiers: List[ast.Identifier] = []
if self._peek_token_is(TokenType.RPAREN):
self._next_token()
return identifiers
self._next_token()
ident = ast.Identifier(self._current_token,
self._current_token.literal)
identifiers.append(ident)
while self._peek_token_is(TokenType.COMMA):
self._next_token()
self._next_token()
ident = ast.Identifier(self._current_token,
self._current_token.literal)
identifiers.append(ident)
if not self._expect_peek(TokenType.RPAREN):
return None
return identifiers
def parseFunctionParameters(self):
parameters = []
if self.peekTokenIs(tkn.RPAREN):
self.nextToken()
return parameters
self.nextToken()
ident = ast.Identifier(self.currToken, self.currToken.Literal)
parameters.append(ident)
while self.peekTokenIs(tkn.COMMA):
self.nextToken()
self.nextToken()
ident = ast.Identifier(self.currToken, self.currToken.Literal)
parameters.append(ident)
if not self.expectedPeekToken(tkn.RPAREN):
return None
return parameters
def parse_function_call(self, first=None):
expr = ast.FunctionCall(self.cur_tok, [])
if first != None:
expr.pattern.append(first)
# If the current token is a valid arg token, or a keyword
while self.peek_in(self.arg_tokens) or self.peek_in(
token.keywords.values()):
self.next()
if self.cur_in(token.keywords.values()):
expr.pattern.append(ast.Identifier(self.cur_tok))
continue
arg = lambda val: ast.Argument(self.cur_tok, val)
handlers = {
token.ID: lambda: ast.Identifier(self.cur_tok),
token.PARAM: lambda: arg(ast.Identifier(self.cur_tok))
}
found = False
for k, v in self.prefixes.items():
if k in handlers.keys():
continue
if k == self.cur_tok.type:
expr.pattern.append(arg(v()))
found = True
if not found:
handler = handlers[self.cur_tok.type]
expr.pattern.append(handler())
if len(expr.pattern) == 0:
self.err("expected at least one item in a pattern")
return None
return expr
def _app(self, name):
self._match("(")
args = []
while self.cur_token.type != ")":
args.append(self._expr())
if self.cur_token.type == ",":
self._get_next_token()
elif self.cur_token.type == ")":
pass
else:
self._error(f"Unexpected {self.cur_token.val} in application")
self._match(")")
return ast.AppExpr(ast.Identifier(name), args)
def parse_function_call(self, first=None):
expr = ast.FunctionCall(self.cur_tok, [])
if first != None:
expr.pattern.append(first)
# If the current token is a valid arg token, or a keyword
while self.peek_in(arg_tokens) or self.peek_in(
token.keywords.values()):
self.next()
if self.cur_in(token.keywords.values()):
expr.pattern.append(ast.Identifier(self.cur_tok))
continue
arg = lambda val: ast.Argument(self.cur_tok, val)
handlers = {
token.ID: lambda: ast.Identifier(self.cur_tok),
token.LPAREN: lambda: arg(self.parse_grouped_expr()),
token.NUM: lambda: arg(self.parse_num()),
token.NULL: lambda: arg(self.parse_null()),
token.TRUE: lambda: arg(self.parse_bool()),
token.FALSE: lambda: arg(self.parse_bool()),
token.STR: lambda: arg(self.parse_string()),
token.PARAM: lambda: arg(ast.Identifier(self.cur_tok)),
token.LSQUARE: lambda: arg(self.parse_array_or_object()),
token.LBRACE: lambda: arg(self.parse_block_literal()),
}
handler = handlers[self.cur_tok.type]
expr.pattern.append(handler())
if len(expr.pattern) == 0:
self.err("expected at least one item in a pattern")
return None
return expr
def _parse_parameter(self):
""" <parameter>
Parses the <parameter> language structure.
parameter -> 'identifer' ':' type_specifier
"""
token = self._look
self._match(lexer.Tag.ID, 'identifier')
self._match(':')
param_type = self._parse_type_specifier()
param_id = ast.Identifier(token, param_type)
return param_id
def _app(self, name):
self._match('(')
args = []
while self.cur_token.type != ')':
args.append(self._expr())
if self.cur_token.type == ',':
self._get_next_token()
elif self.cur_token.type == ')':
pass # the loop will break
else:
self._error("Unexpected {} in application".format(
self.cur_token.val))
self._match(')')
return ast.AppExpr(ast.Identifier(name), args)
def _parse_struct_declaration(self):
""" <struct_declaration>
Parses the <struct_declaration> language structure.
struct_declaration -> 'struct' 'identifier' ';'
"""
self._match(lexer.Tag.STRUCT)
token = self._look
self._match(lexer.Tag.ID)
self._match(';')
id_type = ty.Struct(token)
id_obj = ast.Identifier(token, id_type)
id_type.get_identifier_table().init(prev=None, owner_id=id_obj)
return id_obj
def _parse_variable_declaration(self):
""" <variable_declaration>
Parses the <variable_declaration> language structure.
variable_declaration -> 'identifier' ':' type_specifier ';'
"""
token = self._look
self._match(lexer.Tag.ID)
self._match(':')
oftype = self._parse_type_specifier()
self._match(';')
id_obj = ast.Identifier(token, oftype)
return id_obj
def parse_pattern_call(self, end):
pattern = []
while not self.cur_is(end):
tok = self.cur_tok
if not self.expect_cur_any([token.ID, token.PARAM] +
list(token.keywords.values())):
return None
if tok.type == token.ID or tok.type in token.keywords.values():
pattern.append(ast.Identifier(tok))
else:
pattern.append(ast.Parameter(tok, tok.literal))
return pattern
def ParseLetStatement(self):
letStmt = ast.LetStatement(self.currToken, None, None)
if not self.expectedPeekToken(tkn.IDENT):
return None
letStmt.Name = ast.Identifier(self.currToken, self.currToken.Literal)
if not self.expectedPeekToken(tkn.ASSIGN):
return None
self.nextToken()
letStmt.Value = self.parseExpression(Precedence.LOWEST)
return letStmt
def parseLetStatement(self):
stmt = ast.LetStatement(Token=self.curToken,Name=None,Value=None)
if not self.expectPeek(tokens.IDENT):
return None
stmt.Name = ast.Identifier(Token=self.curToken,Value=self.curToken.Literal)
if not self.expectPeek(tokens.ASSIGN):
return None
self.nextToken()
stmt.Value = self.parseExpression(self.operators.LOWEST)
while not self.curTokenIs(tokens.SEMICOLON):
self.nextToken()
return stmt
def resolveName_FuncDef(self, func):
# self.logger.debug('resolveName_FuncDef', func.name, func, func.owner)
if isinstance(func.owner, ast.ExtensionDef):
func.cls = func.owner.cls
func.owner.cls.addSymbol(func)
owner = func.owner
if func.info.type == ast.FuncType.constructor:
# self.logger.debug('resolveName_FuncDef constructor returnType', func.name, func, func.owner)
assert func.spec.returnType is None
func.spec.returnType = ast.UserType([func.cls.name])
self.setupNewItem(func.spec.returnType, func.spec, False)
assert isinstance(owner, ast.ClassDef) or isinstance(owner, ast.CodeUnit) or isinstance(owner, ast.ExtensionDef), 'resolveName_FuncDef owner is invalid: %s %s' % (owner, func)
if func.info.type == ast.FuncType.constructor:
for var in func.owner.vars:
if var.initial:
stmt = ast.Assignment([ast.Identifier(var.name)], '=', [var.initial])
func.body.statements.insert(0, stmt)
# self.logger.debug('resolveName_FuncDef initialize var', func.name, stmt, var, var.initial)
self.setupNewItem(stmt, func.body, False)
func.visitChildren(self)
def _parse_function_declaration(self):
""" <function_declaration>
Parses the <function_declaration> language structure.
function_declaration -> 'function' 'identifer' '(' parameter_list ')' basic_specifier ';'
"""
self._match(lexer.Tag.FUNCTION, 'function')
token = self._look
self._match(lexer.Tag.ID, 'identifier')
self._match('(')
func_type = ty.Function()
func_proto = self._parse_parameter_list(
func_type.get_identifier_table(), [])
self._match(')')
ret_type = self._parse_basic_specifier()
func_type.init(func_proto, ret_type)
func_id = ast.Identifier(token, func_type)
self._match(';')
return func_id
