def process_fn(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder, lfp: tl.LineFilePos):
abstract = self.abstract
self.abstract = False
const = self.var_level == ast.VAR_CONST
self.var_level = ast.VAR_VAR
permission = self.permission
self.permission = ast.PUBLIC
inline = self.inline
self.inline = False
index += 1
name_list = tl.CollectiveElement(tl.CE_BRACKET, lfp, None)
next_ele = parent[index]
while not tl.is_bracket(next_ele):
name_list.append(next_ele)
index += 1
next_ele = parent[index]
if len(name_list) == 0:
fn_name = None
else:
fn_name = self.parse_as_part(name_list)
param_list = next_ele
params = self.parse_as_line(param_list)
prob_arrow = parent[index + 1]
if tl.identifier_of(prob_arrow, "->"):
builder.add_node(ast.FunctionTypeExpr(params, lfp))
return index
index += 1
rtype_list = tl.CollectiveElement(tl.CE_BRACKET, lfp, None)
while not (tl.is_brace(parent[index]) or tl.identifier_of(parent[index], ";")):
rtype_list.append(parent[index])
index += 1
rtype = self.parse_as_part(rtype_list)
body_list = parent[index]
if tl.identifier_of(body_list, ";"):
body = None
else:
body = self.parse_as_block(body_list)
if not abstract and body is None:
raise errs.TplSyntaxError("Non-abstract method must have body. ", lfp)
if abstract and body is not None:
raise errs.TplSyntaxError("Abstract method must not have body. ", lfp)
fd = ast.FunctionDef(fn_name, params, rtype, abstract, const, permission, body, lfp)
fd.inline = inline
builder.add_node(fd)
return index
def process_annotation(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder,
lfp: tl.LineFilePos):
index += 1
ele = parent[index]
if isinstance(ele, tl.AtomicElement) and isinstance(ele.atom, tl.IdToken):
builder.add_node(ast.AnnotationNode(ele.atom.identifier, lfp))
else:
raise errs.TplSyntaxError("Invalid annotation. ", lfp)
return index
def make_tree(self, cur_active: tl.CollectiveElement,
index: int) -> tl.CollectiveElement:
tk = self.tokens[index]
if isinstance(tk, tl.IdToken):
symbol = tk.identifier
if symbol == "(":
return tl.CollectiveElement(tl.CE_BRACKET, tk.lfp, cur_active)
if symbol == "[":
return tl.CollectiveElement(tl.CE_SQR_BRACKET, tk.lfp,
cur_active)
if symbol == "{":
return tl.CollectiveElement(tl.CE_BRACE, tk.lfp, cur_active)
if symbol == ")":
if cur_active.ce_type == tl.CE_BRACKET:
cur_active.parent.append(cur_active)
return cur_active.parent
else:
raise errs.TplSyntaxError("Parenthesis does not close. ",
tk.lfp)
if symbol == "]":
if cur_active.ce_type == tl.CE_SQR_BRACKET:
cur_active.parent.append(cur_active)
return cur_active.parent
else:
raise errs.TplSyntaxError("Parenthesis does not close. ",
tk.lfp)
if symbol == "}":
if cur_active.ce_type == tl.CE_BRACE:
cur_active.parent.append(cur_active)
return cur_active.parent
else:
raise errs.TplSyntaxError("Parenthesis does not close. ",
tk.lfp)
if symbol == "<":
if has_closing_arrow(self.tokens, index):
return tl.CollectiveElement(tl.CE_ARROW_BRACKET, tk.lfp,
cur_active)
if symbol == ">":
if tl.is_arrow_bracket(cur_active):
cur_active.parent.append(cur_active)
return cur_active.parent
cur_active.append(tl.AtomicElement(tk, cur_active))
return cur_active
def get_name_list(lst: tl.CollectiveElement) -> list:
res = []
for part in lst:
if isinstance(part, tl.AtomicElement) and \
isinstance(part.atom, tl.IdToken) and \
part.atom.identifier.isidentifier():
res.append(part.atom.identifier)
else:
raise errs.TplSyntaxError(
"Macro param list must all be identifiers. ")
return res
def process_lambda(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder, lfp: tl.LineFilePos):
index += 1
params_bracket = parent[index]
if not tl.is_bracket(params_bracket):
raise errs.TplSyntaxError("Invalid syntax for lambda expression. ", lfp)
params = self.parse_as_line(params_bracket)
body_bracket = tl.CollectiveElement(tl.CE_BRACKET, lfp, None)
index += 1
next_tk = parent[index]
while not tl.identifier_of(next_tk, ";"):
body_bracket.append(next_tk)
index += 1
next_tk = parent[index]
body = self.parse_as_part(body_bracket)
builder.add_node(ast.LambdaExpr(params, body, lfp))
return index
def line_tokenize(self, content: str, lf: tl.LineFile,
part_start_pos: int):
lst = normalize_line(content)
length = len(lst)
i = 0
pos = part_start_pos
while i < length:
s = lst[i]
if is_int(s):
if i < length - 2 and lst[i + 1] == "." and is_int(lst[i + 2]):
self.tokens.append(
tl.FloatToken(s + "." + lst[i + 2],
tl.LineFilePos(lf, pos)))
i += 2
elif i < length - 1 and lst[i + 1] == "b":
self.tokens.append(tl.ByteToken(s,
tl.LineFilePos(lf,
pos))) # 1b
i += 1
elif i < length - 1 and lst[i + 1] == "f":
self.tokens.append(
tl.FloatToken(s, tl.LineFilePos(
lf, pos))) # situation like 1f (== 1.0)
i += 1
else:
self.tokens.append(tl.IntToken(s, tl.LineFilePos(lf, pos)))
# elif s == "\n":
# self.tokens.append(tl.IdToken(s, lf))
elif s.isidentifier():
self.tokens.append(tl.IdToken(s, tl.LineFilePos(lf, pos)))
elif s in tl.ALL:
self.tokens.append(tl.IdToken(s, tl.LineFilePos(lf, pos)))
elif len(s.strip()) > 0:
raise errs.TplSyntaxError(f"Unexpected token '{s.strip()}'. ",
tl.LineFilePos(lf, pos))
i += 1
pos += len(s)
def process_inline(self, p, i, b, lfp):
if not valid_comb_till_specific(p, i, {"fn"}, {"private", "protected", "const"}):
raise errs.TplSyntaxError("Invalid keywords combination. ", lfp)
self.inline = True
def process_protected(self, p, i, b, lfp):
if not valid_comb_raw(p, i, {"inline", "const"}, {"protected", "private"}):
raise errs.TplSyntaxError("Invalid keywords combination. ", lfp)
self.permission = ast.PROTECTED
def process_abstract(self, p, i, b, lfp):
if not valid_comb_till_specific(p, i, {"fn", "class"}, {"protected", "private"}):
raise errs.TplSyntaxError("Invalid keywords combination. ", lfp)
self.abstract = True
def process_const(self, p, i, b, lfp):
if not valid_comb_raw(p, i, set(), {"const", "var"}):
raise errs.TplSyntaxError("Invalid keywords combination. ", lfp)
self.var_level = ast.VAR_CONST
def process_one(self, parent: tl.CollectiveElement, index: int,
result_parent: tl.CollectiveElement) -> int:
ele = parent[index]
if isinstance(ele, tl.AtomicElement):
if isinstance(ele.atom, tl.IdToken):
symbol = ele.atom.identifier
lf = ele.atom.lfp
if symbol == "macro":
index += 1
name_ele = parent[index]
if isinstance(name_ele, tl.AtomicElement) and isinstance(
name_ele.atom, tl.IdToken):
name = name_ele.atom.identifier
index += 1
prob_params = parent[index]
if isinstance(prob_params, tl.CollectiveElement):
if prob_params.is_brace(): # macro name { ... }
macro = SimpleMacro(prob_params, lf)
self.macros.add_macro(name, macro, lf)
return index + 1
elif prob_params.is_bracket(
): # macro name(...) ...
index += 1
body = parent[index]
if tl.is_brace(body):
macro = CallMacro(
get_name_list(prob_params), body, lf)
self.macros.add_macro(name, macro, lf)
return index + 1
raise errs.TplSyntaxError("Invalid macro syntax. ", lf)
elif symbol == "exportmacro":
index += 1
exports = parent[index]
if tl.is_brace(exports):
for exp in exports:
if isinstance(exp,
tl.AtomicElement) and isinstance(
exp.atom, tl.IdToken):
if exp.atom.identifier != ",":
self.macros.add_export(exp.atom.identifier)
else:
raise errs.TplSyntaxError(
"Invalid exportmacro syntax. ", lf)
elif isinstance(exports, tl.AtomicElement) and isinstance(
exports.atom, tl.IdToken):
self.macros.add_export(exports.atom.identifier)
else:
raise errs.TplSyntaxError(
"Invalid exportmacro syntax. ", lf)
return index + 1
elif symbol == "import":
index += 1
includes = parent[index]
if isinstance(includes, tl.AtomicElement):
self.import_one(includes.atom, result_parent, lf)
elif tl.is_brace(includes):
for inc in includes:
if isinstance(inc, tl.AtomicElement):
self.import_one(inc.atom, result_parent, lf)
else:
raise errs.TplSyntaxError(
"Invalid include. ", lf)
else:
raise errs.TplSyntaxError("Invalid include. ", lf)
return index + 1
elif self.macros.is_macro(symbol): # replace macro
macro = self.macros.get_macro(symbol)
if isinstance(macro, CallMacro):
index += 1
args = parent[index]
if not tl.is_bracket(args):
raise errs.TplSyntaxError("Invalid macro syntax. ",
args.lfp)
arg_list = list_ify_macro_arg(args)
if len(arg_list) != len(macro.params):
raise errs.TplSyntaxError(
"Macro syntax arity mismatch. ", args.lfp)
body_with_arg = tl.CollectiveElement(
tl.CE_BRACE, lf, None)
for body_ele in macro.body:
if isinstance(body_ele, tl.AtomicElement) and \
isinstance(body_ele.atom, tl.IdToken) and \
body_ele.atom.identifier in macro.params:
arg_index = macro.params.index(
body_ele.atom.identifier)
body_with_arg.extend(arg_list[arg_index])
else:
body_with_arg.append(body_ele)
macro_res = self.process_block(body_with_arg, None)
else:
macro_res = self.process_block(macro.body, None)
result_parent.extend(macro_res)
return index + 1
result_parent.append(ele)
elif isinstance(ele, tl.CollectiveElement):
res = self.process_block(ele, result_parent)
result_parent.append(res)
else:
raise errs.TplError("Unexpected element. ", parent.lfp)
return index + 1
def proceed_line(self, content: str, lf: tl.LineFile):
in_str = False
length = len(content)
literal = ""
non_literal = ""
i = 0
part_start_pos = 0
while i < length:
ch = content[i]
if self.in_doc:
if i < length - 1 and ch == '*' and content[i + 1] == '/':
# exit doc
self.in_doc = False
i += 2
else:
if in_str:
if ch == '"':
in_str = False
self.tokens.append(
tl.StrToken(literal,
tl.LineFilePos(lf, part_start_pos)))
literal = ""
part_start_pos = i
else:
literal += ch
else:
if i < length - 1 and ch == '/' and content[i + 1] == '*':
# enter doc
self.in_doc = True
i += 1
elif i < length - 1 and ch == '/' and content[i +
1] == '/':
# enter comment, end of this line
if len(non_literal) > 2:
self.line_tokenize(
non_literal[0:len(non_literal) - 2], lf,
part_start_pos)
non_literal = ""
part_start_pos = i
break
elif ch == '"':
# enter string literal
in_str = True
self.line_tokenize(non_literal, lf, part_start_pos)
non_literal = ""
part_start_pos = i
elif ch == '\'':
# enter char literal
if i < length - 2 and content[i + 2] == '\'':
# normal char
self.line_tokenize(non_literal, lf, part_start_pos)
non_literal = ""
part_start_pos = i
self.tokens.append(
tl.CharToken(
content[i + 1],
tl.LineFilePos(lf, part_start_pos)))
i += 2
elif i < length - 3 and content[
i + 3] == '\'' and content[i + 1] == '\\':
# escape char
self.line_tokenize(non_literal, lf, part_start_pos)
non_literal = ""
part_start_pos = i
escaped = content[i + 2]
if escaped == '\\':
self.tokens.append(
tl.CharToken(
'\\',
tl.LineFilePos(lf, part_start_pos)))
elif escaped in ESCAPES:
self.tokens.append(
tl.CharToken(
ESCAPES[escaped],
tl.LineFilePos(lf, part_start_pos)))
else:
raise errs.TplSyntaxError(
"Invalid escape '\\" + escaped + "'. ",
tl.LineFilePos(lf, part_start_pos))
i += 3
else:
raise errs.TplSyntaxError(
"Char literal must contain exactly one symbol. ",
tl.LineFilePos(lf, part_start_pos))
else:
non_literal += ch
i += 1
if len(non_literal) > 0:
self.line_tokenize(non_literal, lf, part_start_pos)
