def parse_added_lines(added_lines, method_name): tmp = '\n'.join([line for lineno, line in added_lines]) seq = split_to_str(tmp) seq = tokenizer.parse(seq) seq = search_method(method_name).parse(seq) seq = seq_split_nodes_of_label(seq, "null")[0] if len(list(seq_outermost_node_iter(seq, 'target_method'))) == 0: return [] for expression in parsing_expressions: seq = expression.parse(seq) seq = seq_split_nodes_of_label(seq, "null")[0] num_args_list = set() for pos, invoke_seq in seq_outermost_node_iter(seq, 'method_invoke'): params = len(list(seq_outermost_node_iter(invoke_seq, 'param'))) num_args_list.add(params) return num_args_list
def main(debugTrace=False): import sys if len(sys.argv) == 1: print "usage: tinyawk -f <script> [ <input> ]\nAn interpreter of a awk-like small language." return assert len(sys.argv) in (3, 4) assert sys.argv[1] == "-f" scriptFile = sys.argv[2] inputFile = sys.argv[3] if len(sys.argv) == 4 else None debugWrite = sys.stderr.write if debugTrace else None f = open(scriptFile, "r") try: script = f.read() finally: f.close() script = script + "\n" # prepare for missing new-line char at the last line # parsing seq = split_to_str(script) des = [] des.extend(tokenizing_expr_iter()) des.extend(stmt_parsing_expr_iter()) des.extend(expr_parsing_expr_iter()) for desc, expr in des: if debugWrite: debugWrite("\n".join(treeseq.seq_pretty(treeseq.seq_remove_strattrs(seq))) + "\n") # prints a seq debugWrite("step: %s\n" % desc) newSeq = expr.parse(seq) if newSeq is None: sys.exit("parse error") seq = newSeq seq = seq_split_nodes_of_label(seq, "null")[0] if debugWrite: debugWrite("\n".join(treeseq.seq_pretty(treeseq.seq_remove_strattrs(seq))) + "\n") # prints a seq seq = treeseq.seq_remove_strattrs(seq) # interpretation interp = StmtInterpreter(seq) def dbgwrite(): if debugWrite: debugWrite("variables=%s\n" % repr(interp.varTable)) interp.apply_begin(); dbgwrite() if interp.expects_input(): f = open(inputFile, "r") if inputFile else sys.stdin for lnum, L in enumerate(f): interp.apply_line(lnum+1, L.rstrip()); dbgwrite() if inputFile: f.close() interp.apply_end(); dbgwrite()
def tokenize(tokenizer, script): seq = split_to_str(script) seq = tokenizer.parse(seq) return seq_split_nodes_of_label(seq, "null")[0]
op = itemQ.popleft() right = interpret_i(itemQ.popleft()) value = _opstr_to_func[op](value, right) return value return interpret_i(ast) if __name__ == '__main__': import sys if len(sys.argv) == 1: print(""" usage: calculator <expr> A simple calculator. <expr> must consist of floating-point numbers and operators: +,-,*,/,%,(). For example, "1 + 2 * (-3) - 4 * (5 - 6) * -7". """[1:-1]) sys.exit(0) ts = pyrem_torq.treeseq text = " ".join(sys.argv[1:]) seq = tokenize(text) for expr in parsing_exprs: for L in ts.seq_pretty(ts.seq_remove_strattrs(seq)): print L # prints an seq newSeq = expr.parse(seq) if newSeq is None: sys.exit("Syntax Error") seq = newSeq seq = seq_split_nodes_of_label(seq, "null")[0] for L in ts.seq_pretty(ts.seq_remove_strattrs(seq)): print L # prints an seq result = interpret(ts.seq_remove_strattrs(seq)) print result
def parse_to_ast(inputSeq, verboseOutput=None): A = _pte.Any AN = _pte.AnyNode BtN = _pte.BuildToNode IN = _pte.InsertNode L = _pte.Literal def LC(strs): return _pte.Or(*map(_pte.Literal, strs)) # LC = _pte.LiteralClass N = _pte.Node NM = _pte.NodeMatch R = _pte.Rex XtA = _pte.AnyBut Holder = _pte.Holder def markNull(expr): return BtN("null", expr) def flattened(nodeExpr): return _pte.Flattened(nodeExpr) def relabeled(newLabel, nodeExpr): return _pte.Relabeled(newLabel, nodeExpr) seq = [] if verboseOutput: @contextmanager def verbose_print_step_title_and_result_seq(title): verboseOutput.write(title) try: yield finally: verboseOutput.write("\n".join(seq_pretty(seq))); verboseOutput.write("\n") else: @contextmanager def verbose_print_step_title_and_result_seq(title): yield with verbose_print_step_title_and_result_seq('input'): pat = re.compile(r"\d+|[a-z_][a-z_0-9]*|\r\n|.", re.DOTALL | re.IGNORECASE) s = ['code'] + split_to_strings(inputSeq, pat) seq[:] = s with verbose_print_step_title_and_result_seq('ParseToken'): def parseTokenExpr(): def rst(label, *strings): assert len(strings) >= 1 e = L(strings[0]) for s in strings[1:]: e = e + L(s) return BtN(label, e) tokenExpr = _pte.Or( # spaces/comments markNull(BtN("space", [1, ] * R(r"^\s$"))), markNull(BtN("comment", L('#') + [0, ] * XtA(LC(["\n", "\r", "\r\n"])))), # operators rst("insert_subtree", "<", "-"), rst("comma", ","), rst("semicolon", ";"), rst("matches", ":", ":"), rst("anybut", "any", "^"), rst("reqbut", "req", "^"), rst("LP", "("), rst("RP", ")"), rst("joinplus", "+", "+"), rst("joinstar", "*", "*"), rst("plus", "+"), rst("ques", "?"), rst("star", "*"), rst("or", "|"), rst("diamond", "[", "]"), rst("search", "~"), # string literal BtN("string_literal", [0, 1] * LC(['i', 'ir', 'r', 'ri']) + L('"') + [0, ] * (L('\\') + XtA(LC(['\t', '\n', '\r', '\f', '\v'])) | \ XtA(LC(['"', '\t', '\n', '\r', '\f', '\v'])) ) + L('"')), L('"') + _pte.ErrorExpr('Invalid string literal'), # identifier (or reserved word) BtN("id", L('_') + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]")) | R(r"^[a-zA-Z]") + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]"))), # marker BtN('marker', markNull(BtN('markerop', L('@'))) + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]"))) | L('@') + _pte.ErrorExpr('Invalid marker name') ) return __fill(tokenExpr, "Can't extract a token") seq[:] = parseTokenExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseReservedWords'): def parseReservedWordsExpr(): def rw(name): return relabeled(name, NM('id', L(name))) # reserved word reservedWordExpr = rw('req') | rw('error') | rw('any') return _pte.Search(reservedWordExpr) seq[:] = parseReservedWordsExpr().parse(seq) with verbose_print_step_title_and_result_seq('preChecking'): def parsePreCheckingExpr(): idOrStr = N('id') | N('string_literal') preCheckExpr = _pte.Or( idOrStr + idOrStr + _pte.ErrorExpr("expected ',' between two ids/string literals"), N('RP') + idOrStr + _pte.ErrorExpr("expected ',' after paren"), idOrStr + N('LP') + _pte.ErrorExpr("expected ',' before paren (or misspelled built-in operator?)"), N('RP') + N('LP') + _pte.ErrorExpr("expected ',' after closing paren, before opening paren"), N('semicolon') + N('semicolon') + _pte.ErrorExpr("';' appears just after ';'"), L('^') + _pte.ErrorExpr("unexpected '^' (or misspelling of 'any^' or 'req^'?)"), ) return _pte.Search(preCheckExpr) seq[:] = parsePreCheckingExpr().parse(seq) with verbose_print_step_title_and_result_seq('parseParen'): def parseParenExpr(): parenExpr = Holder('parenExpr') parenExpr.expr = XtA((N('LP') | N('RP'))) \ | BtN('apply', IN('insert_subtree') + markNull(N('LP')) + (N('id') | N('null')) + markNull(N('insert_subtree')) + markNull(N('RP'))) \ | BtN('param', markNull(N('LP')) + [0, ] * parenExpr + markNull(N('RP'))) return __fill(parenExpr.expr, "Can't parse parens") seq[:] = parseParenExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] def recurseApplyAndParam(marker): return NM('apply', A() + [0, ] * marker) | NM('param', [0, ] * marker) with verbose_print_step_title_and_result_seq('parseParen'): def parseUnaryOperatorsExpr(): unaryOperatorExpr = Holder('unaryOperatorExpr') unaryOperatorExpr.expr = _pte.Or(recurseApplyAndParam(unaryOperatorExpr), BtN('apply', (N("plus") | N("ques") | N("star") | N('search') | N('reqbut') | N('req') | N('anybut')) + unaryOperatorExpr), BtN('error', markNull(N('error')) + flattened(N('string_literal'))), BtN('error', markNull(N('error')) + \ flattened(NM('param', flattened(N('string_literal'))))), N('diamond') + N('id') + N('matches'), # special form BtN('apply', IN("expand") + markNull(N('diamond')) + N('id')), N('diamond') + _pte.ErrorExpr('operator [] only applicable to a node'), A()) return _pte.Search(unaryOperatorExpr.expr) seq[:] = parseUnaryOperatorsExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorJoin'): def parseBinaryOperatorJoinExpr(): joinExpr = Holder('joinExpr') term = recurseApplyAndParam(joinExpr) | XtA((N('joinplus') | N('joinstar'))) joinExpr.expr = BtN('apply', IN('joinplus') + term + markNull(N('joinplus')) + joinExpr) \ | BtN('apply', IN('joinstar') + term + markNull(N('joinstar')) + joinExpr) \ | term return __fill(joinExpr.expr, "Can't parse binary operator join (++, **)") seq[:] = parseBinaryOperatorJoinExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorSeq'): def parseBinaryOperatorSeqExpr(): seqExpr = Holder('seqExpr') term = recurseApplyAndParam(seqExpr) | XtA(N('comma')) seqExpr.expr = BtN('apply', IN('seq') + term + [1, ] * (markNull(N('comma')) + term)) \ | term return __fill(seqExpr, "Can't parse binary operator Seq (,)") seq[:] = parseBinaryOperatorSeqExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorOr'): def parseBinaryOperatorOrExpr(): orExpr = Holder('orExpr') term = recurseApplyAndParam(orExpr) | XtA(N('or')) orExpr.expr = BtN('apply', IN('or') + term + [1, ] * (markNull(N('or')) + term)) \ | term return __fill(orExpr.expr, "Can't parse binary operator Or (|)") seq[:] = parseBinaryOperatorOrExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorMatch'): def parseBinaryOperatorMatchExpr(): matchExpr = Holder('matchExpr') def aopwd(opName): return BtN('apply', IN(opName) + IN('expand') + markNull(N('diamond')) + N('id') + markNull(N(opName)) + matchExpr) term = recurseApplyAndParam(matchExpr) | XtA((N('matches') | N('assign_subtree'))) matchExpr.expr = aopwd('matches') \ | BtN('apply', IN('matches') + N('id') + markNull(N('matches')) + matchExpr) \ | BtN('apply', IN('insert_subtree') + (N('id') | N('null')) + markNull(N('insert_subtree')) + matchExpr) \ | term return __fill(matchExpr.expr, "Can't parse binary operator match (::)") seq[:] = parseBinaryOperatorMatchExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseReduceRedundantParen'): def parseReduceRedundantParenExpr(): paramExpr = Holder('paramExpr') term = NM('apply', A() + [0, ] * paramExpr) | XtA(N('param')) paramExpr.expr = flattened(NM('param', paramExpr)) \ | NM('param', AN()) \ | term return __fill(paramExpr.expr, "Can't parse redundant paren") seq[:] = parseReduceRedundantParenExpr().parse(seq) with verbose_print_step_title_and_result_seq('parseStatement'): def parseStatementExpr(): statementExpr = BtN('statement', N('semicolon') | XtA(N('semicolon')) + N('semicolon')) return statementExpr + (_pte.EndOfNode() | _pte.ErrorExpr("Can't parse statement")) seq[:] = parseStatementExpr().parse(seq) #print "\n".join(seq_pretty(seq)) return seq