def process_fn(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder, lfp: tl.LineFilePos): abstract = self.abstract self.abstract = False const = self.var_level == ast.VAR_CONST self.var_level = ast.VAR_VAR permission = self.permission self.permission = ast.PUBLIC inline = self.inline self.inline = False index += 1 name_list = tl.CollectiveElement(tl.CE_BRACKET, lfp, None) next_ele = parent[index] while not tl.is_bracket(next_ele): name_list.append(next_ele) index += 1 next_ele = parent[index] if len(name_list) == 0: fn_name = None else: fn_name = self.parse_as_part(name_list) param_list = next_ele params = self.parse_as_line(param_list) prob_arrow = parent[index + 1] if tl.identifier_of(prob_arrow, "->"): builder.add_node(ast.FunctionTypeExpr(params, lfp)) return index index += 1 rtype_list = tl.CollectiveElement(tl.CE_BRACKET, lfp, None) while not (tl.is_brace(parent[index]) or tl.identifier_of(parent[index], ";")): rtype_list.append(parent[index]) index += 1 rtype = self.parse_as_part(rtype_list) body_list = parent[index] if tl.identifier_of(body_list, ";"): body = None else: body = self.parse_as_block(body_list) if not abstract and body is None: raise errs.TplSyntaxError("Non-abstract method must have body. ", lfp) if abstract and body is not None: raise errs.TplSyntaxError("Abstract method must not have body. ", lfp) fd = ast.FunctionDef(fn_name, params, rtype, abstract, const, permission, body, lfp) fd.inline = inline builder.add_node(fd) return index
def process_annotation(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder, lfp: tl.LineFilePos): index += 1 ele = parent[index] if isinstance(ele, tl.AtomicElement) and isinstance(ele.atom, tl.IdToken): builder.add_node(ast.AnnotationNode(ele.atom.identifier, lfp)) else: raise errs.TplSyntaxError("Invalid annotation. ", lfp) return index
def make_tree(self, cur_active: tl.CollectiveElement, index: int) -> tl.CollectiveElement: tk = self.tokens[index] if isinstance(tk, tl.IdToken): symbol = tk.identifier if symbol == "(": return tl.CollectiveElement(tl.CE_BRACKET, tk.lfp, cur_active) if symbol == "[": return tl.CollectiveElement(tl.CE_SQR_BRACKET, tk.lfp, cur_active) if symbol == "{": return tl.CollectiveElement(tl.CE_BRACE, tk.lfp, cur_active) if symbol == ")": if cur_active.ce_type == tl.CE_BRACKET: cur_active.parent.append(cur_active) return cur_active.parent else: raise errs.TplSyntaxError("Parenthesis does not close. ", tk.lfp) if symbol == "]": if cur_active.ce_type == tl.CE_SQR_BRACKET: cur_active.parent.append(cur_active) return cur_active.parent else: raise errs.TplSyntaxError("Parenthesis does not close. ", tk.lfp) if symbol == "}": if cur_active.ce_type == tl.CE_BRACE: cur_active.parent.append(cur_active) return cur_active.parent else: raise errs.TplSyntaxError("Parenthesis does not close. ", tk.lfp) if symbol == "<": if has_closing_arrow(self.tokens, index): return tl.CollectiveElement(tl.CE_ARROW_BRACKET, tk.lfp, cur_active) if symbol == ">": if tl.is_arrow_bracket(cur_active): cur_active.parent.append(cur_active) return cur_active.parent cur_active.append(tl.AtomicElement(tk, cur_active)) return cur_active
def get_name_list(lst: tl.CollectiveElement) -> list: res = [] for part in lst: if isinstance(part, tl.AtomicElement) and \ isinstance(part.atom, tl.IdToken) and \ part.atom.identifier.isidentifier(): res.append(part.atom.identifier) else: raise errs.TplSyntaxError( "Macro param list must all be identifiers. ") return res
def process_lambda(self, parent: tl.CollectiveElement, index: int, builder: ab.AstBuilder, lfp: tl.LineFilePos): index += 1 params_bracket = parent[index] if not tl.is_bracket(params_bracket): raise errs.TplSyntaxError("Invalid syntax for lambda expression. ", lfp) params = self.parse_as_line(params_bracket) body_bracket = tl.CollectiveElement(tl.CE_BRACKET, lfp, None) index += 1 next_tk = parent[index] while not tl.identifier_of(next_tk, ";"): body_bracket.append(next_tk) index += 1 next_tk = parent[index] body = self.parse_as_part(body_bracket) builder.add_node(ast.LambdaExpr(params, body, lfp)) return index
def line_tokenize(self, content: str, lf: tl.LineFile, part_start_pos: int): lst = normalize_line(content) length = len(lst) i = 0 pos = part_start_pos while i < length: s = lst[i] if is_int(s): if i < length - 2 and lst[i + 1] == "." and is_int(lst[i + 2]): self.tokens.append( tl.FloatToken(s + "." + lst[i + 2], tl.LineFilePos(lf, pos))) i += 2 elif i < length - 1 and lst[i + 1] == "b": self.tokens.append(tl.ByteToken(s, tl.LineFilePos(lf, pos))) # 1b i += 1 elif i < length - 1 and lst[i + 1] == "f": self.tokens.append( tl.FloatToken(s, tl.LineFilePos( lf, pos))) # situation like 1f (== 1.0) i += 1 else: self.tokens.append(tl.IntToken(s, tl.LineFilePos(lf, pos))) # elif s == "\n": # self.tokens.append(tl.IdToken(s, lf)) elif s.isidentifier(): self.tokens.append(tl.IdToken(s, tl.LineFilePos(lf, pos))) elif s in tl.ALL: self.tokens.append(tl.IdToken(s, tl.LineFilePos(lf, pos))) elif len(s.strip()) > 0: raise errs.TplSyntaxError(f"Unexpected token '{s.strip()}'. ", tl.LineFilePos(lf, pos)) i += 1 pos += len(s)
def process_inline(self, p, i, b, lfp): if not valid_comb_till_specific(p, i, {"fn"}, {"private", "protected", "const"}): raise errs.TplSyntaxError("Invalid keywords combination. ", lfp) self.inline = True
def process_protected(self, p, i, b, lfp): if not valid_comb_raw(p, i, {"inline", "const"}, {"protected", "private"}): raise errs.TplSyntaxError("Invalid keywords combination. ", lfp) self.permission = ast.PROTECTED
def process_abstract(self, p, i, b, lfp): if not valid_comb_till_specific(p, i, {"fn", "class"}, {"protected", "private"}): raise errs.TplSyntaxError("Invalid keywords combination. ", lfp) self.abstract = True
def process_const(self, p, i, b, lfp): if not valid_comb_raw(p, i, set(), {"const", "var"}): raise errs.TplSyntaxError("Invalid keywords combination. ", lfp) self.var_level = ast.VAR_CONST
def process_one(self, parent: tl.CollectiveElement, index: int, result_parent: tl.CollectiveElement) -> int: ele = parent[index] if isinstance(ele, tl.AtomicElement): if isinstance(ele.atom, tl.IdToken): symbol = ele.atom.identifier lf = ele.atom.lfp if symbol == "macro": index += 1 name_ele = parent[index] if isinstance(name_ele, tl.AtomicElement) and isinstance( name_ele.atom, tl.IdToken): name = name_ele.atom.identifier index += 1 prob_params = parent[index] if isinstance(prob_params, tl.CollectiveElement): if prob_params.is_brace(): # macro name { ... } macro = SimpleMacro(prob_params, lf) self.macros.add_macro(name, macro, lf) return index + 1 elif prob_params.is_bracket( ): # macro name(...) ... index += 1 body = parent[index] if tl.is_brace(body): macro = CallMacro( get_name_list(prob_params), body, lf) self.macros.add_macro(name, macro, lf) return index + 1 raise errs.TplSyntaxError("Invalid macro syntax. ", lf) elif symbol == "exportmacro": index += 1 exports = parent[index] if tl.is_brace(exports): for exp in exports: if isinstance(exp, tl.AtomicElement) and isinstance( exp.atom, tl.IdToken): if exp.atom.identifier != ",": self.macros.add_export(exp.atom.identifier) else: raise errs.TplSyntaxError( "Invalid exportmacro syntax. ", lf) elif isinstance(exports, tl.AtomicElement) and isinstance( exports.atom, tl.IdToken): self.macros.add_export(exports.atom.identifier) else: raise errs.TplSyntaxError( "Invalid exportmacro syntax. ", lf) return index + 1 elif symbol == "import": index += 1 includes = parent[index] if isinstance(includes, tl.AtomicElement): self.import_one(includes.atom, result_parent, lf) elif tl.is_brace(includes): for inc in includes: if isinstance(inc, tl.AtomicElement): self.import_one(inc.atom, result_parent, lf) else: raise errs.TplSyntaxError( "Invalid include. ", lf) else: raise errs.TplSyntaxError("Invalid include. ", lf) return index + 1 elif self.macros.is_macro(symbol): # replace macro macro = self.macros.get_macro(symbol) if isinstance(macro, CallMacro): index += 1 args = parent[index] if not tl.is_bracket(args): raise errs.TplSyntaxError("Invalid macro syntax. ", args.lfp) arg_list = list_ify_macro_arg(args) if len(arg_list) != len(macro.params): raise errs.TplSyntaxError( "Macro syntax arity mismatch. ", args.lfp) body_with_arg = tl.CollectiveElement( tl.CE_BRACE, lf, None) for body_ele in macro.body: if isinstance(body_ele, tl.AtomicElement) and \ isinstance(body_ele.atom, tl.IdToken) and \ body_ele.atom.identifier in macro.params: arg_index = macro.params.index( body_ele.atom.identifier) body_with_arg.extend(arg_list[arg_index]) else: body_with_arg.append(body_ele) macro_res = self.process_block(body_with_arg, None) else: macro_res = self.process_block(macro.body, None) result_parent.extend(macro_res) return index + 1 result_parent.append(ele) elif isinstance(ele, tl.CollectiveElement): res = self.process_block(ele, result_parent) result_parent.append(res) else: raise errs.TplError("Unexpected element. ", parent.lfp) return index + 1
def proceed_line(self, content: str, lf: tl.LineFile): in_str = False length = len(content) literal = "" non_literal = "" i = 0 part_start_pos = 0 while i < length: ch = content[i] if self.in_doc: if i < length - 1 and ch == '*' and content[i + 1] == '/': # exit doc self.in_doc = False i += 2 else: if in_str: if ch == '"': in_str = False self.tokens.append( tl.StrToken(literal, tl.LineFilePos(lf, part_start_pos))) literal = "" part_start_pos = i else: literal += ch else: if i < length - 1 and ch == '/' and content[i + 1] == '*': # enter doc self.in_doc = True i += 1 elif i < length - 1 and ch == '/' and content[i + 1] == '/': # enter comment, end of this line if len(non_literal) > 2: self.line_tokenize( non_literal[0:len(non_literal) - 2], lf, part_start_pos) non_literal = "" part_start_pos = i break elif ch == '"': # enter string literal in_str = True self.line_tokenize(non_literal, lf, part_start_pos) non_literal = "" part_start_pos = i elif ch == '\'': # enter char literal if i < length - 2 and content[i + 2] == '\'': # normal char self.line_tokenize(non_literal, lf, part_start_pos) non_literal = "" part_start_pos = i self.tokens.append( tl.CharToken( content[i + 1], tl.LineFilePos(lf, part_start_pos))) i += 2 elif i < length - 3 and content[ i + 3] == '\'' and content[i + 1] == '\\': # escape char self.line_tokenize(non_literal, lf, part_start_pos) non_literal = "" part_start_pos = i escaped = content[i + 2] if escaped == '\\': self.tokens.append( tl.CharToken( '\\', tl.LineFilePos(lf, part_start_pos))) elif escaped in ESCAPES: self.tokens.append( tl.CharToken( ESCAPES[escaped], tl.LineFilePos(lf, part_start_pos))) else: raise errs.TplSyntaxError( "Invalid escape '\\" + escaped + "'. ", tl.LineFilePos(lf, part_start_pos)) i += 3 else: raise errs.TplSyntaxError( "Char literal must contain exactly one symbol. ", tl.LineFilePos(lf, part_start_pos)) else: non_literal += ch i += 1 if len(non_literal) > 0: self.line_tokenize(non_literal, lf, part_start_pos)