def parse_added_lines(added_lines, method_name):
tmp = '\n'.join([line for lineno, line in added_lines])
seq = split_to_str(tmp)
seq = tokenizer.parse(seq)
seq = search_method(method_name).parse(seq)
seq = seq_split_nodes_of_label(seq, "null")[0]
if len(list(seq_outermost_node_iter(seq, 'target_method'))) == 0:
return []
for expression in parsing_expressions:
seq = expression.parse(seq)
seq = seq_split_nodes_of_label(seq, "null")[0]
num_args_list = set()
for pos, invoke_seq in seq_outermost_node_iter(seq, 'method_invoke'):
params = len(list(seq_outermost_node_iter(invoke_seq, 'param')))
num_args_list.add(params)
return num_args_list
def parse_added_lines(added_lines, method_name):
tmp = '\n'.join([line for lineno, line in added_lines])
seq = split_to_str(tmp)
seq = tokenizer.parse(seq)
seq = search_method(method_name).parse(seq)
seq = seq_split_nodes_of_label(seq, "null")[0]
if len(list(seq_outermost_node_iter(seq, 'target_method'))) == 0:
return []
for expression in parsing_expressions:
seq = expression.parse(seq)
seq = seq_split_nodes_of_label(seq, "null")[0]
num_args_list = set()
for pos, invoke_seq in seq_outermost_node_iter(seq, 'method_invoke'):
params = len(list(seq_outermost_node_iter(invoke_seq, 'param')))
num_args_list.add(params)
return num_args_list
def main(debugTrace=False):
import sys
if len(sys.argv) == 1:
print "usage: tinyawk -f <script> [ <input> ]\nAn interpreter of a awk-like small language."
return
assert len(sys.argv) in (3, 4)
assert sys.argv[1] == "-f"
scriptFile = sys.argv[2]
inputFile = sys.argv[3] if len(sys.argv) == 4 else None
debugWrite = sys.stderr.write if debugTrace else None
f = open(scriptFile, "r")
try:
script = f.read()
finally: f.close()
script = script + "\n" # prepare for missing new-line char at the last line
# parsing
seq = split_to_str(script)
des = []
des.extend(tokenizing_expr_iter())
des.extend(stmt_parsing_expr_iter())
des.extend(expr_parsing_expr_iter())
for desc, expr in des:
if debugWrite:
debugWrite("\n".join(treeseq.seq_pretty(treeseq.seq_remove_strattrs(seq))) + "\n") # prints a seq
debugWrite("step: %s\n" % desc)
newSeq = expr.parse(seq)
if newSeq is None: sys.exit("parse error")
seq = newSeq
seq = seq_split_nodes_of_label(seq, "null")[0]
if debugWrite: debugWrite("\n".join(treeseq.seq_pretty(treeseq.seq_remove_strattrs(seq))) + "\n") # prints a seq
seq = treeseq.seq_remove_strattrs(seq)
# interpretation
interp = StmtInterpreter(seq)
def dbgwrite():
if debugWrite: debugWrite("variables=%s\n" % repr(interp.varTable))
interp.apply_begin(); dbgwrite()
if interp.expects_input():
f = open(inputFile, "r") if inputFile else sys.stdin
for lnum, L in enumerate(f):
interp.apply_line(lnum+1, L.rstrip()); dbgwrite()
if inputFile: f.close()
interp.apply_end(); dbgwrite()
def tokenize(tokenizer, script):
seq = split_to_str(script)
seq = tokenizer.parse(seq)
return seq_split_nodes_of_label(seq, "null")[0]
def tokenize(tokenizer, script):
seq = split_to_str(script)
seq = tokenizer.parse(seq)
return seq_split_nodes_of_label(seq, "null")[0]
op = itemQ.popleft()
right = interpret_i(itemQ.popleft())
value = _opstr_to_func[op](value, right)
return value
return interpret_i(ast)
if __name__ == '__main__':
import sys
if len(sys.argv) == 1:
print("""
usage: calculator <expr>
A simple calculator. <expr> must consist of floating-point numbers and
operators: +,-,*,/,%,(). For example, "1 + 2 * (-3) - 4 * (5 - 6) * -7".
"""[1:-1])
sys.exit(0)
ts = pyrem_torq.treeseq
text = " ".join(sys.argv[1:])
seq = tokenize(text)
for expr in parsing_exprs:
for L in ts.seq_pretty(ts.seq_remove_strattrs(seq)): print L # prints an seq
newSeq = expr.parse(seq)
if newSeq is None: sys.exit("Syntax Error")
seq = newSeq
seq = seq_split_nodes_of_label(seq, "null")[0]
for L in ts.seq_pretty(ts.seq_remove_strattrs(seq)): print L # prints an seq
result = interpret(ts.seq_remove_strattrs(seq))
print result
def parse_to_ast(inputSeq, verboseOutput=None):
A = _pte.Any
AN = _pte.AnyNode
BtN = _pte.BuildToNode
IN = _pte.InsertNode
L = _pte.Literal
def LC(strs): return _pte.Or(*map(_pte.Literal, strs)) # LC = _pte.LiteralClass
N = _pte.Node
NM = _pte.NodeMatch
R = _pte.Rex
XtA = _pte.AnyBut
Holder = _pte.Holder
def markNull(expr): return BtN("null", expr)
def flattened(nodeExpr): return _pte.Flattened(nodeExpr)
def relabeled(newLabel, nodeExpr): return _pte.Relabeled(newLabel, nodeExpr)
seq = []
if verboseOutput:
@contextmanager
def verbose_print_step_title_and_result_seq(title):
verboseOutput.write(title)
try: yield
finally:
verboseOutput.write("\n".join(seq_pretty(seq))); verboseOutput.write("\n")
else:
@contextmanager
def verbose_print_step_title_and_result_seq(title):
yield
with verbose_print_step_title_and_result_seq('input'):
pat = re.compile(r"\d+|[a-z_][a-z_0-9]*|\r\n|.", re.DOTALL | re.IGNORECASE)
s = ['code'] + split_to_strings(inputSeq, pat)
seq[:] = s
with verbose_print_step_title_and_result_seq('ParseToken'):
def parseTokenExpr():
def rst(label, *strings):
assert len(strings) >= 1
e = L(strings[0])
for s in strings[1:]: e = e + L(s)
return BtN(label, e)
tokenExpr = _pte.Or(
# spaces/comments
markNull(BtN("space", [1, ] * R(r"^\s$"))),
markNull(BtN("comment", L('#') + [0, ] * XtA(LC(["\n", "\r", "\r\n"])))),
# operators
rst("insert_subtree", "<", "-"),
rst("comma", ","), rst("semicolon", ";"),
rst("matches", ":", ":"), rst("anybut", "any", "^"), rst("reqbut", "req", "^"),
rst("LP", "("), rst("RP", ")"),
rst("joinplus", "+", "+"), rst("joinstar", "*", "*"),
rst("plus", "+"), rst("ques", "?"), rst("star", "*"), rst("or", "|"), rst("diamond", "[", "]"),
rst("search", "~"),
# string literal
BtN("string_literal", [0, 1] * LC(['i', 'ir', 'r', 'ri']) + L('"') + [0, ] * (L('\\') + XtA(LC(['\t', '\n', '\r', '\f', '\v'])) | \
XtA(LC(['"', '\t', '\n', '\r', '\f', '\v']))
) + L('"')),
L('"') + _pte.ErrorExpr('Invalid string literal'),
# identifier (or reserved word)
BtN("id", L('_') + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]")) |
R(r"^[a-zA-Z]") + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]"))),
# marker
BtN('marker',
markNull(BtN('markerop', L('@'))) + [0, ] * (L('_') | R(r"^[a-zA-Z]") | R(r"^[0-9]")))
| L('@') + _pte.ErrorExpr('Invalid marker name')
)
return __fill(tokenExpr, "Can't extract a token")
seq[:] = parseTokenExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseReservedWords'):
def parseReservedWordsExpr():
def rw(name): return relabeled(name, NM('id', L(name))) # reserved word
reservedWordExpr = rw('req') | rw('error') | rw('any')
return _pte.Search(reservedWordExpr)
seq[:] = parseReservedWordsExpr().parse(seq)
with verbose_print_step_title_and_result_seq('preChecking'):
def parsePreCheckingExpr():
idOrStr = N('id') | N('string_literal')
preCheckExpr = _pte.Or(
idOrStr + idOrStr + _pte.ErrorExpr("expected ',' between two ids/string literals"),
N('RP') + idOrStr + _pte.ErrorExpr("expected ',' after paren"),
idOrStr + N('LP') + _pte.ErrorExpr("expected ',' before paren (or misspelled built-in operator?)"),
N('RP') + N('LP') + _pte.ErrorExpr("expected ',' after closing paren, before opening paren"),
N('semicolon') + N('semicolon') + _pte.ErrorExpr("';' appears just after ';'"),
L('^') + _pte.ErrorExpr("unexpected '^' (or misspelling of 'any^' or 'req^'?)"),
)
return _pte.Search(preCheckExpr)
seq[:] = parsePreCheckingExpr().parse(seq)
with verbose_print_step_title_and_result_seq('parseParen'):
def parseParenExpr():
parenExpr = Holder('parenExpr')
parenExpr.expr = XtA((N('LP') | N('RP'))) \
| BtN('apply', IN('insert_subtree') + markNull(N('LP')) + (N('id') | N('null')) + markNull(N('insert_subtree')) + markNull(N('RP'))) \
| BtN('param', markNull(N('LP')) + [0, ] * parenExpr + markNull(N('RP')))
return __fill(parenExpr.expr, "Can't parse parens")
seq[:] = parseParenExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
def recurseApplyAndParam(marker): return NM('apply', A() + [0, ] * marker) | NM('param', [0, ] * marker)
with verbose_print_step_title_and_result_seq('parseParen'):
def parseUnaryOperatorsExpr():
unaryOperatorExpr = Holder('unaryOperatorExpr')
unaryOperatorExpr.expr = _pte.Or(recurseApplyAndParam(unaryOperatorExpr),
BtN('apply', (N("plus") | N("ques") | N("star") | N('search') | N('reqbut') | N('req') | N('anybut')) + unaryOperatorExpr),
BtN('error', markNull(N('error')) + flattened(N('string_literal'))),
BtN('error', markNull(N('error')) + \
flattened(NM('param', flattened(N('string_literal'))))),
N('diamond') + N('id') + N('matches'), # special form
BtN('apply', IN("expand") + markNull(N('diamond')) + N('id')),
N('diamond') + _pte.ErrorExpr('operator [] only applicable to a node'),
A())
return _pte.Search(unaryOperatorExpr.expr)
seq[:] = parseUnaryOperatorsExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseBinaryOperatorJoin'):
def parseBinaryOperatorJoinExpr():
joinExpr = Holder('joinExpr')
term = recurseApplyAndParam(joinExpr) | XtA((N('joinplus') | N('joinstar')))
joinExpr.expr = BtN('apply', IN('joinplus') + term + markNull(N('joinplus')) + joinExpr) \
| BtN('apply', IN('joinstar') + term + markNull(N('joinstar')) + joinExpr) \
| term
return __fill(joinExpr.expr, "Can't parse binary operator join (++, **)")
seq[:] = parseBinaryOperatorJoinExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseBinaryOperatorSeq'):
def parseBinaryOperatorSeqExpr():
seqExpr = Holder('seqExpr')
term = recurseApplyAndParam(seqExpr) | XtA(N('comma'))
seqExpr.expr = BtN('apply', IN('seq') + term + [1, ] * (markNull(N('comma')) + term)) \
| term
return __fill(seqExpr, "Can't parse binary operator Seq (,)")
seq[:] = parseBinaryOperatorSeqExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseBinaryOperatorOr'):
def parseBinaryOperatorOrExpr():
orExpr = Holder('orExpr')
term = recurseApplyAndParam(orExpr) | XtA(N('or'))
orExpr.expr = BtN('apply',
IN('or') + term + [1, ] * (markNull(N('or')) + term)) \
| term
return __fill(orExpr.expr, "Can't parse binary operator Or (|)")
seq[:] = parseBinaryOperatorOrExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseBinaryOperatorMatch'):
def parseBinaryOperatorMatchExpr():
matchExpr = Holder('matchExpr')
def aopwd(opName): return BtN('apply', IN(opName) + IN('expand') + markNull(N('diamond')) + N('id') + markNull(N(opName)) + matchExpr)
term = recurseApplyAndParam(matchExpr) | XtA((N('matches') | N('assign_subtree')))
matchExpr.expr = aopwd('matches') \
| BtN('apply', IN('matches') + N('id') + markNull(N('matches')) + matchExpr) \
| BtN('apply', IN('insert_subtree') + (N('id') | N('null')) + markNull(N('insert_subtree')) + matchExpr) \
| term
return __fill(matchExpr.expr, "Can't parse binary operator match (::)")
seq[:] = parseBinaryOperatorMatchExpr().parse(seq)
seq[:] = seq_split_nodes_of_label(seq, "null")[0]
with verbose_print_step_title_and_result_seq('parseReduceRedundantParen'):
def parseReduceRedundantParenExpr():
paramExpr = Holder('paramExpr')
term = NM('apply', A() + [0, ] * paramExpr) | XtA(N('param'))
paramExpr.expr = flattened(NM('param', paramExpr)) \
| NM('param', AN()) \
| term
return __fill(paramExpr.expr, "Can't parse redundant paren")
seq[:] = parseReduceRedundantParenExpr().parse(seq)
with verbose_print_step_title_and_result_seq('parseStatement'):
def parseStatementExpr():
statementExpr = BtN('statement', N('semicolon') | XtA(N('semicolon')) + N('semicolon'))
return statementExpr + (_pte.EndOfNode() | _pte.ErrorExpr("Can't parse statement"))
seq[:] = parseStatementExpr().parse(seq)
#print "\n".join(seq_pretty(seq))
return seq
