def testParseExpressionInStatement(self): kit = OperatorBuilder() kit.atomic_term_expr = Rex(r"^\d") kit.composed_term_node_labels = ( "t", ) kit.generated_term_label = "t" descAndExprs = [] descAndExprs.append(( "atomic", kit.build_atom_to_term_expr() )) descAndExprs.append(( "paren", kit.build_O_expr(( Literal("("), Literal(")") )) )) descAndExprs.append(( "sign", kit.build_Ot_expr(Literal("+"), Literal("-")) )) descAndExprs.append(( "mul,div", kit.build_tOt_expr(Literal("*"), Literal("/")) )) descAndExprs.append(( "add,sub", kit.build_tOt_expr(Literal("+"), Literal("-")) )) def exprssionParser(seq): seq.insert(0, 'code') for desc, expr in descAndExprs: seq = expr.parse(seq) del seq[0] return seq exprsionToken = Rex(r"^\d") | Or(*map(Literal, [ "+", "-", "*", "/", "(", ")" ])) expressionExpr = SubApply(exprssionParser, Repeat(exprsionToken, 0, None)) statementExpr = Holder() statementExpr.expr = BuildToNode("s", Literal("print") + expressionExpr + Literal(";") | \ Literal("if") + expressionExpr + Literal("then") + statementExpr + Literal("else") + statementExpr) removeSpaces = lambda seq: [] if len(seq) == 2 and seq[1] == ' ' else seq spaceRemoveExpr = SubApply(removeSpaces, Repeat(Any(), 0, None)) text = "print 1*(3+4);" seq = [ 'code' ] + split_to_strings(text, re.compile(r"[a-z]+|(\d|[.])+|[-+*/%()?:,;]|\[|\]")) seq = spaceRemoveExpr.parse(seq) seq = statementExpr.parse(seq) seq_wo_attr = seq_remove_strattrs(seq) print "\n".join(seq_pretty(seq_wo_attr)) self.assertTrue(seq_wo_attr[1][0] == 's') s = seq_wo_attr[1] self.assertTrue(s[1] == 'print') self.assertTrue(s[2][0] == 't') self.assertTrue(s[3] == ';') text = "if -1 then print -1; else print 1;" seq = [ 'code' ] + split_to_strings(text, re.compile(r"[a-z]+|(\d|[.])+|[-+*/%()?:,;]|\[|\]")) seq = spaceRemoveExpr.parse(seq) seq = statementExpr.parse(seq) seq_wo_attr = seq_remove_strattrs(seq) print "\n".join(seq_pretty(seq_wo_attr)) self.assertTrue(seq_wo_attr[1][0] == 's') s = seq_wo_attr[1] self.assertTrue(s[1] == 'if') self.assertTrue(s[3] == 'then') self.assertTrue(s[5] == 'else')
def test9th(self): exprs = compile_exprs([ r""" ~("a", error "literal 'a' should not appear" | any); """ ]) assert len(exprs) == 1 inputText = r'b,c,d' seq = [ 'code' ] + split_to_strings(inputText) seq = exprs[0].parse(seq) self.assertEqual(seq, [ 'code', 0, 'b', 1, ',', 2, 'c', 3, ',', 4, 'd' ]) inputText = r'a,b,c' seq = [ 'code' ] + split_to_strings(inputText) self.assertRaises(pyrem_torq.expression.InterpretErrorByErrorExpr, exprs[0].parse, seq)
def testParseExpression(self): kit = OperatorBuilder() kit.atomic_term_expr = Rex(r"^\d") | Rex(r"^\w") kit.composed_term_node_labels = ( "t", ) kit.generated_term_label = "t" descAndExprs = [] descAndExprs.append(( "atomicExpr", kit.build_atom_to_term_expr() )) descAndExprs.append(( "funcCallExpr", kit.build_tO_expr(( BuildToNode("CL", Literal("(")), BuildToNode("CR", Literal(")")) ), pseudoPrefix="TO") )) descAndExprs.append(( "parenExpr", kit.build_O_expr(( BuildToNode("PL", Literal("(")), BuildToNode("PR", Literal(")")) )) )) descAndExprs.append(( "indexExpr", kit.build_tO_expr(( Literal("["), Literal("]") )) )) descAndExprs.append(( "unaryMinusExpr", kit.build_Ot_expr(Literal("-")) )) descAndExprs.append(( "binaryStarExpr", kit.build_tOt_expr(Literal("*")) )) descAndExprs.append(( "binaryMinusExpr", kit.build_tOt_expr(Literal("-")) )) descAndExprs.append(( "conditionExpr", kit.build_tOt_expr(( Literal("?"), Literal(":") ), pseudoPrefix="TOT") )) results = map(scan_seq, [ r"[code,-,[t,1],-,[t,2],*,(,[t,3],-,[t,4],),-,[t,a],\[,[t,5],\],*,[t,6],?,[t,7],:,[t,8],-,[t,9],?,[t,b],(,[t,c],-,[t,1],\,,[t,d],),:,[t,e]]", r"[code,-,[t,1],-,[t,2],*,(,[t,3],-,[t,4],),-,[t,a],\[,[t,5],\],*,[t,6],?,[t,7],:,[t,8],-,[t,9],?,[t,[TO],[t,b],[CL,(],[t,c],-,[t,1],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,-,[t,1],-,[t,2],*,[t,[PL,(],[t,3],-,[t,4],[PR,)]],-,[t,a],\[,[t,5],\],*,[t,6],?,[t,7],:,[t,8],-,[t,9],?,[t,[TO],[t,b],[CL,(],[t,c],-,[t,1],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,-,[t,1],-,[t,2],*,[t,[PL,(],[t,3],-,[t,4],[PR,)]],-,[t,[t,a],\[,[t,5],\]],*,[t,6],?,[t,7],:,[t,8],-,[t,9],?,[t,[TO],[t,b],[CL,(],[t,c],-,[t,1],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,[t,-,[t,1]],-,[t,2],*,[t,[PL,(],[t,3],-,[t,4],[PR,)]],-,[t,[t,a],\[,[t,5],\]],*,[t,6],?,[t,7],:,[t,8],-,[t,9],?,[t,[TO],[t,b],[CL,(],[t,c],-,[t,1],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,[t,-,[t,1]],-,[t,[t,2],*,[t,[PL,(],[t,3],-,[t,4],[PR,)]]],-,[t,[t,[t,a],\[,[t,5],\]],*,[t,6]],?,[t,7],:,[t,8],-,[t,9],?,[t,[TO],[t,b],[CL,(],[t,c],-,[t,1],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,[t,[t,-,[t,1]],-,[t,[t,2],*,[t,[PL,(],[t,[t,3],-,[t,4]],[PR,)]]],-,[t,[t,[t,a],\[,[t,5],\]],*,[t,6]]],?,[t,7],:,[t,[t,8],-,[t,9]],?,[t,[TO],[t,b],[CL,(],[t,[t,c],-,[t,1]],\,,[t,d],[CR,)]],:,[t,e]]", r"[code,[t,[TOT],[t,[t,-,[t,1]],-,[t,[t,2],*,[t,[PL,(],[t,[t,3],-,[t,4]],[PR,)]]],-,[t,[t,[t,a],\[,[t,5],\]],*,[t,6]]],?,[t,7],:,[t,[t,8],-,[t,9]],?,[t,[TO],[t,b],[CL,(],[t,[t,c],-,[t,1]],\,,[t,d],[CR,)]],:,[t,e]]]", ]) text = "-1-2*(3-4)-a[5]*6?7:8-9?b(c-1,d):e" seq = [ 'code' ] + split_to_strings(text, re.compile(r"[a-z]+|(\d|[.])+|[-+*/%()?:,]|\[|\]")) for ( desc, expr ), result in zip(descAndExprs, results): print "step: %s" % desc seq = expr.parse(seq) seq_wo_attr = seq_remove_strattrs(seq) sys.stdout.write("\n".join(seq_pretty(seq_wo_attr)) + "\n") self.assertEquals(seq_wo_attr, result)
def test3rd(self): exprStrs = [ r'~(eol <- "\t" | "\f" | "\v" | "\r" | "\n");', r'~(comment <- "/", "*", *(+"*", (req^("/"), any) | req^("*"), any), +"*", "/");', r'~(comment <- "/", "/", *(req^(eol), any));', ] exprs = compile_exprs(exprStrs) inputText = """ #include <stdio.h> // import printf() int main(int argc, char *argv[]) { /************************ ** the arguments argc/** ** argv are not used. ** ************************/ printf("hello, world.\n"); return 0; } """[1:-1] seq = [ 'code' ] + split_to_strings(inputText) for expr in exprs: posDelta, outSeq = expr.match(seq, 1) self.assertEqual(1 + posDelta, len(seq)) seq = [ seq[0] ] + outSeq print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test1st(self): exprStrs = [ r'~((v <- +(r"^\d" | ".")) | (null <- +(" " | "\t")));', r'~(v <- (null <- "("), +(@0 | req^("(" | ")"), any), (null <- ")"));', r""" ?(v <- (u_op <- "+" | "-"), (v :: ~@0)), *( (v :: ~@0), *("+" | "-") | (v <- (u_op <- "+" | "-"), (v :: ~@0)) | any ); """, r'~((v <- (v :: ~@0), +((b_op <- "**"), (v :: ~@0))) | (v :: ~@0));', r'~((v <- (v :: ~@0), +((b_op <- "*" | "/"), (v :: ~@0))) | (v :: ~@0));', r'~((v <- (v :: ~@0), +((b_op <- "+" | "-"), (v :: ~@0))) | (v :: ~@0));', ] exprs = compile_exprs(exprStrs) seq = [ 'code' ] + split_to_strings("+1.0 + 2 * ((3 - 4) / -.5) ** 6") for exprIndex, expr in enumerate(exprs): print "exprIndex=", exprIndex, "cur seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq)) newSeq = expr.parse(seq) self.assertTrue(newSeq, None) seq = newSeq print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def testJoining(self): exprs = compile_exprs([ 'expr <- ","++"b";', 'expr <- ","**"b";' ]) assert len(exprs) == 2 inputText = r'b,b,b' seq = [ 'code' ] + split_to_strings(inputText) r0 = exprs[0].parse(seq) self.assertEqual(r0, [ 'code', [ 'expr', 0, 'b', 1, ',', 2, 'b', 3, ',', 4, 'b' ] ]) r1 = exprs[1].parse(seq) self.assertEqual(r1, [ 'code', [ 'expr', 0, 'b', 1, ',', 2, 'b', 3, ',', 4, 'b' ] ])
def test6th(self): searchWordLike = r'~(wordlike <- "_", *(r"^[a-zA-Z]" | r"\d" | "_") | r"^[a-zA-Z]", *(r"^[a-zA-Z]" | r"\d" | "_"));' exprs = compile_exprs([ searchWordLike ]) assert len(exprs) == 1 inputText = r'argv[0];' inputText = inputText.decode(sys.getfilesystemencoding()) seq = [ 'code' ] + split_to_strings(inputText) seq = exprs[0].parse(seq) self.assertEqual(seq[1], [ 'wordlike', 0, u'argv' ]) print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test4th(self): IN = pyrem_torq.expression.InsertNode BtN = pyrem_torq.expression.BuildToNode L = pyrem_torq.expression.Literal A = pyrem_torq.expression.Any Q = pyrem_torq.expression.Require S = pyrem_torq.expression.Search eolExpr = BtN('eol', L("\r\n") | L("\n") | L("\r")) expr = S(eolExpr) + Q(pyrem_torq.expression.EndOfNode()) + IN('eof') seq = [ 'code' ] + split_to_strings("abc\n") posDelta, outSeq = expr.match(seq, 1) self.assertEqual(1 + posDelta, 5) self.assertEqual(outSeq, [ 0, 'abc', [ 'eol', 3, '\n' ], [ 'eof' ] ])
def test7th(self): searchWordLike = r'~(word <- ("_", *("_" | r"^[a-zA-Z]" | r"^\d") | r"^[a-zA-Z]", *("_" | r"^[a-zA-Z]" | r"^\d")));' searchIdMake = r'~(id <- []word);' exprs = compile_exprs([ searchWordLike, searchIdMake ]) assert len(exprs) == 2 inputText = r'argv[0];' inputText = inputText.decode(sys.getfilesystemencoding()) seq = [ 'code' ] + split_to_strings(inputText) for expr in exprs: newSeq = expr.parse(seq) self.assertTrue(newSeq) seq = newSeq print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def testParseExpressionWithGerbageToken(self): kit = OperatorBuilder() kit.atomic_term_expr = Rex(r"^\d") kit.composed_term_node_labels = ( "t", ) kit.generated_term_label = "t" descAndExprs = [] descAndExprs.append(( "atomicExpr", kit.build_atom_to_term_expr() )) descAndExprs.append(( "funcCallExpr", kit.build_tO_expr(( BuildToNode("CL", Literal("(")), BuildToNode("CR", Literal(")")) ), pseudoPrefix="TO") )) descAndExprs.append(( "parenExpr", kit.build_O_expr(( BuildToNode("PL", Literal("(")), BuildToNode("PR", Literal(")")) )) )) descAndExprs.append(( "unaryMinusExpr", kit.build_Ot_expr(Literal("-")) )) descAndExprs.append(( "binaryStarExpr", kit.build_tOt_expr(Literal("*")) )) descAndExprs.append(( "binaryMinusExpr", kit.build_tOt_expr(Literal("-")) )) text = "-1-2*(3-gerbage)-4" seq = [ 'code' ] + split_to_strings(text, re.compile(r"[a-z]+|(\d|[.])+|[-+*/%()?:,]|\[|\]")) for desc, expr in descAndExprs: #print "step: %s" % desc seq = expr.parse(seq) seq_wo_attr = seq_remove_strattrs(seq)
def test8th(self): searchFloatingPointLitearal = r"""~( l_float <- "0", ( ri"^x[a-f0-9]+p\d+$" | ri"^x[a-f0-9]+p$", ("-" | "+"), r"^\d" | ri"^x[a-f0-9]+$", ".", *(ri"^[a-f0-9]+$" | r"^\d"), ?(ri"^[a-f0-9]*p\d+$" | ri"^[a-f0-9]*p$", ("-" | "+"), r"^\d") ), ?i"l" ); """ exprs = compile_exprs([ searchFloatingPointLitearal ]) assert len(exprs) == 1 pat = re.compile(r"\d+|[a-zA-Z_][a-zA-Z_0-9]*|[ \t]+|\r\n|.", re.DOTALL | re.IGNORECASE) sampleFlotingPointLiterals = [ '0x012abc.def', '0xabc.012', '0xap10' ] for inputText in sampleFlotingPointLiterals: inputText = inputText.decode(sys.getfilesystemencoding()) seq = [ 'code' ] + split_to_strings(inputText, pat) seq = exprs[0].parse(seq) self.assertEqual(seq[0], 'code') self.assertEqual(seq[1][0], 'l_float') self.assertEqual(u"".join(seq[1][2::2]), inputText)
def test5th(self): atoz = 'r"^[a-z]"' exprStrs = [ r'~(req(%(atoz)s), ((op_logical_and <- "and") | (op_logical_or <- "or")), req^(%(atoz)s));' % { 'atoz': atoz } ] exprs = compile_exprs(exprStrs) inputText = r'if (x and y or z) printf("hello\n");' inputText = inputText.decode(sys.getfilesystemencoding()) seq = [ 'code' ] + split_to_strings(inputText) for expr in exprs: newSeq = expr.parse(seq) self.assertTrue(newSeq, None) seq = newSeq foundAnd, foundOr = False, False for item in seq: if isinstance(item, list): if item[0:1] == [ 'op_logical_and' ]: foundAnd = True if item[0:1] == [ 'op_logical_or' ]: foundOr = True self.assertTrue(foundAnd) self.assertTrue(foundOr) print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
expr = Or(*ovpExprs) expr0.expr = expr | termExpr return Search(expr0) if __name__ == '__main__': import re from pyrem_torq.treeseq import seq_pretty, seq_remove_strattrs from pyrem_torq.utility import split_to_strings kit = OperatorBuilder() kit.atomic_term_expr = Rex(r"^\d") | Rex(r"^\w") kit.composed_term_node_labels = ("t", ) kit.generated_term_label = "t" descAndExprs = [] #descAndExprs.append(( "atomicExpr", kit.build_atom_to_term_expr() )) descAndExprs.append(("funcCallExpr", kit.build_tO_expr((BuildToNode("CL", Literal("(")), BuildToNode("CR", Literal(")"))), pseudoPrefix="TO"))) descAndExprs.append(("parenExpr", kit.build_O_expr((BuildToNode("PL", Literal("(")), BuildToNode("PR", Literal(")")))))) descAndExprs.append(("indexExpr", kit.build_tO_expr((Literal("["), Literal("]"))))) descAndExprs.append(("unaryMinusExpr", kit.build_Ot_expr(Literal("-")))) descAndExprs.append(("binaryStarExpr", kit.build_tOt_expr(Literal("*")))) descAndExprs.append(("binaryMinusExpr", kit.build_tOt_expr(Literal("-")))) descAndExprs.append(("conditionExpr", kit.build_tOt_expr((Literal("?"), Literal(":")), pseudoPrefix="TOT"))) text = "-1-2*(3-4)-a[5]*6?7:8-9?b(c-1,d):e" seq = ['code'] + split_to_strings(text, re.compile(r"[a-z]+|(\d|[.])+|[-+*/%()?:,]|\[|\]")) for desc, expr in descAndExprs: print "step: %s" % desc seq = expr.parse(seq) for L in seq_pretty(seq_remove_strattrs(seq)): print L
def parse_to_ast(inputSeq, verboseOutput=None): A = _pte.Any AN = _pte.AnyNode BtN = _pte.BuildToNode IN = _pte.InsertNode L = _pte.Literal def LC(strs): return _pte.Or(*map(_pte.Literal, strs)) # LC = _pte.LiteralClass N = _pte.Node NM = _pte.NodeMatch R = _pte.Rex XtA = _pte.AnyBut Holder = _pte.Holder def markNull(expr): return BtN("null", expr) def flattened(nodeExpr): return _pte.Flattened(nodeExpr) def relabeled(newLabel, nodeExpr): return _pte.Relabeled(newLabel, nodeExpr) seq = [] if verboseOutput: @contextmanager def verbose_print_step_title_and_result_seq(title): verboseOutput.write(title) try: yield finally: verboseOutput.write("\n".join(seq_pretty(seq))); verboseOutput.write("\n") else: @contextmanager def verbose_print_step_title_and_result_seq(title): yield with verbose_print_step_title_and_result_seq('input'): pat = re.compile(r"\d+|[a-z_][a-z_0-9]*|\r\n|.", re.DOTALL | re.IGNORECASE) s = ['code'] + split_to_strings(inputSeq, pat) seq[:] = s with verbose_print_step_title_and_result_seq('ParseToken'): def parseTokenExpr(): def rst(label, *strings): assert len(strings) >= 1 e = L(strings[0]) for s in strings[1:]: e = e + L(s) return BtN(label, e) tokenExpr = _pte.Or( # spaces/comments markNull(BtN("space", [1, ] * R(r"^\s$"))), markNull(BtN("comment", L('#') + [0, ] * XtA(LC(["\n", "\r", "\r\n"])))), # operators rst("insert_subtree", "<", "-"), rst("comma", ","), rst("semicolon", ";"), rst("matches", ":", ":"), rst("anybut", "any", "^"), rst("reqbut", "req", "^"), rst("LP", "("), rst("RP", ")"), rst("joinplus", "+", "+"), rst("joinstar", "*", "*"), rst("plus", "+"), rst("ques", "?"), rst("star", "*"), rst("or", "|"), rst("diamond", "[", "]"), rst("search", "~"), # string literal BtN("string_literal", [0, 1] * LC(['i', 'ir', 'r', 'ri']) + L('"') + [0, ] * (L('\\') + XtA(LC(['\t', '\n', '\r', '\f', '\v'])) | \ XtA(LC(['"', '\t', '\n', '\r', '\f', '\v'])) ) + L('"')), L('"') + _pte.ErrorExpr('Invalid string literal'), # identifier (or reserved word) BtN("id", L('_') + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]")) | R(r"^[a-zA-Z]") + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]"))), # marker BtN('marker', markNull(BtN('markerop', L('@'))) + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]"))) | L('@') + _pte.ErrorExpr('Invalid marker name') ) return __fill(tokenExpr, "Can't extract a token") seq[:] = parseTokenExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseReservedWords'): def parseReservedWordsExpr(): def rw(name): return relabeled(name, NM('id', L(name))) # reserved word reservedWordExpr = rw('req') | rw('error') | rw('any') return _pte.Search(reservedWordExpr) seq[:] = parseReservedWordsExpr().parse(seq) with verbose_print_step_title_and_result_seq('preChecking'): def parsePreCheckingExpr(): idOrStr = N('id') | N('string_literal') preCheckExpr = _pte.Or( idOrStr + idOrStr + _pte.ErrorExpr("expected ',' between two ids/string literals"), N('RP') + idOrStr + _pte.ErrorExpr("expected ',' after paren"), idOrStr + N('LP') + _pte.ErrorExpr("expected ',' before paren (or misspelled built-in operator?)"), N('RP') + N('LP') + _pte.ErrorExpr("expected ',' after closing paren, before opening paren"), N('semicolon') + N('semicolon') + _pte.ErrorExpr("';' appears just after ';'"), L('^') + _pte.ErrorExpr("unexpected '^' (or misspelling of 'any^' or 'req^'?)"), ) return _pte.Search(preCheckExpr) seq[:] = parsePreCheckingExpr().parse(seq) with verbose_print_step_title_and_result_seq('parseParen'): def parseParenExpr(): parenExpr = Holder('parenExpr') parenExpr.expr = XtA((N('LP') | N('RP'))) \ | BtN('apply', IN('insert_subtree') + markNull(N('LP')) + (N('id') | N('null')) + markNull(N('insert_subtree')) + markNull(N('RP'))) \ | BtN('param', markNull(N('LP')) + [0, ] * parenExpr + markNull(N('RP'))) return __fill(parenExpr.expr, "Can't parse parens") seq[:] = parseParenExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] def recurseApplyAndParam(marker): return NM('apply', A() + [0, ] * marker) | NM('param', [0, ] * marker) with verbose_print_step_title_and_result_seq('parseParen'): def parseUnaryOperatorsExpr(): unaryOperatorExpr = Holder('unaryOperatorExpr') unaryOperatorExpr.expr = _pte.Or(recurseApplyAndParam(unaryOperatorExpr), BtN('apply', (N("plus") | N("ques") | N("star") | N('search') | N('reqbut') | N('req') | N('anybut')) + unaryOperatorExpr), BtN('error', markNull(N('error')) + flattened(N('string_literal'))), BtN('error', markNull(N('error')) + \ flattened(NM('param', flattened(N('string_literal'))))), N('diamond') + N('id') + N('matches'), # special form BtN('apply', IN("expand") + markNull(N('diamond')) + N('id')), N('diamond') + _pte.ErrorExpr('operator [] only applicable to a node'), A()) return _pte.Search(unaryOperatorExpr.expr) seq[:] = parseUnaryOperatorsExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorJoin'): def parseBinaryOperatorJoinExpr(): joinExpr = Holder('joinExpr') term = recurseApplyAndParam(joinExpr) | XtA((N('joinplus') | N('joinstar'))) joinExpr.expr = BtN('apply', IN('joinplus') + term + markNull(N('joinplus')) + joinExpr) \ | BtN('apply', IN('joinstar') + term + markNull(N('joinstar')) + joinExpr) \ | term return __fill(joinExpr.expr, "Can't parse binary operator join (++, **)") seq[:] = parseBinaryOperatorJoinExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorSeq'): def parseBinaryOperatorSeqExpr(): seqExpr = Holder('seqExpr') term = recurseApplyAndParam(seqExpr) | XtA(N('comma')) seqExpr.expr = BtN('apply', IN('seq') + term + [1, ] * (markNull(N('comma')) + term)) \ | term return __fill(seqExpr, "Can't parse binary operator Seq (,)") seq[:] = parseBinaryOperatorSeqExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorOr'): def parseBinaryOperatorOrExpr(): orExpr = Holder('orExpr') term = recurseApplyAndParam(orExpr) | XtA(N('or')) orExpr.expr = BtN('apply', IN('or') + term + [1, ] * (markNull(N('or')) + term)) \ | term return __fill(orExpr.expr, "Can't parse binary operator Or (|)") seq[:] = parseBinaryOperatorOrExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseBinaryOperatorMatch'): def parseBinaryOperatorMatchExpr(): matchExpr = Holder('matchExpr') def aopwd(opName): return BtN('apply', IN(opName) + IN('expand') + markNull(N('diamond')) + N('id') + markNull(N(opName)) + matchExpr) term = recurseApplyAndParam(matchExpr) | XtA((N('matches') | N('assign_subtree'))) matchExpr.expr = aopwd('matches') \ | BtN('apply', IN('matches') + N('id') + markNull(N('matches')) + matchExpr) \ | BtN('apply', IN('insert_subtree') + (N('id') | N('null')) + markNull(N('insert_subtree')) + matchExpr) \ | term return __fill(matchExpr.expr, "Can't parse binary operator match (::)") seq[:] = parseBinaryOperatorMatchExpr().parse(seq) seq[:] = seq_split_nodes_of_label(seq, "null")[0] with verbose_print_step_title_and_result_seq('parseReduceRedundantParen'): def parseReduceRedundantParenExpr(): paramExpr = Holder('paramExpr') term = NM('apply', A() + [0, ] * paramExpr) | XtA(N('param')) paramExpr.expr = flattened(NM('param', paramExpr)) \ | NM('param', AN()) \ | term return __fill(paramExpr.expr, "Can't parse redundant paren") seq[:] = parseReduceRedundantParenExpr().parse(seq) with verbose_print_step_title_and_result_seq('parseStatement'): def parseStatementExpr(): statementExpr = BtN('statement', N('semicolon') | XtA(N('semicolon')) + N('semicolon')) return statementExpr + (_pte.EndOfNode() | _pte.ErrorExpr("Can't parse statement")) seq[:] = parseStatementExpr().parse(seq) #print "\n".join(seq_pretty(seq)) return seq