def annotated_tokens(code):
prev_was_dot = False
token_source = TokenSource(code)
for (token_type, token) in token_source:
if token_type == tokenize.ENDMARKER:
break
props = {}
props["bare_ref"] = not prev_was_dot and token_type == tokenize.NAME
props["bare_funcall"] = props["bare_ref"] and token_source.peek()[1] == "("
yield (token_type, token, props)
prev_was_dot = token == "."
def annotated_tokens(code):
prev_was_dot = False
token_source = TokenSource(code)
for (token_type, token) in token_source:
if token_type == tokenize.ENDMARKER:
break
props = {}
props["bare_ref"] = (not prev_was_dot and token_type == tokenize.NAME)
props["bare_funcall"] = (props["bare_ref"]
and token_source.peek()[1] == "(")
yield (token_type, token, props)
prev_was_dot = (token == ".")
def parse(code, extra_operators=[]):
code = code.replace("\n", " ").strip()
if not code:
code = "~ 1"
token_source = TokenSource(code)
for extra_operator in extra_operators:
if extra_operator.precedence < 0:
raise ValueError, "all operators must have precedence >= 0"
all_op_list = _default_ops + extra_operators
unary_ops = {}
binary_ops = {}
for op in all_op_list:
if op.arity == 1:
unary_ops[op.token] = op
elif op.arity == 2:
binary_ops[op.token] = op
else:
raise ValueError, "operators must be unary or binary"
c = _ParseContext(unary_ops, binary_ops)
# This is an implementation of Dijkstra's shunting yard algorithm:
# http://en.wikipedia.org/wiki/Shunting_yard_algorithm
# http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm
want_noun = True
while True:
if want_noun:
want_noun = _read_noun_context(token_source, c)
else:
if token_source.peek()[0] == tokenize.ENDMARKER:
break
want_noun = _read_op_context(token_source, c)
while c.op_stack:
if c.op_stack[-1].token == "(":
raise CharltonError("Unmatched '('", c.op_stack[-1])
_run_op(c)
assert len(c.noun_stack) == 1
tree = c.noun_stack.pop()
if not isinstance(tree, ParseNode) or tree.op.token != "~":
tree = ParseNode(unary_ops["~"], [tree], tree.origin)
return tree
def parse(code, extra_operators=[]):
code = code.replace("\n", " ").strip()
if not code:
code = "~ 1"
token_source = TokenSource(code)
for extra_operator in extra_operators:
if extra_operator.precedence < 0:
raise ValueError, "all operators must have precedence >= 0"
all_op_list = _default_ops + extra_operators
unary_ops = {}
binary_ops = {}
for op in all_op_list:
if op.arity == 1:
unary_ops[op.token] = op
elif op.arity == 2:
binary_ops[op.token] = op
else:
raise ValueError, "operators must be unary or binary"
c = _ParseContext(unary_ops, binary_ops)
# This is an implementation of Dijkstra's shunting yard algorithm:
# http://en.wikipedia.org/wiki/Shunting_yard_algorithm
# http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm
want_noun = True
while True:
if want_noun:
want_noun = _read_noun_context(token_source, c)
else:
if token_source.peek()[0] == tokenize.ENDMARKER:
break
want_noun = _read_op_context(token_source, c)
while c.op_stack:
if c.op_stack[-1].token == "(":
raise CharltonError("Unmatched '('", c.op_stack[-1])
_run_op(c)
assert len(c.noun_stack) == 1
tree = c.noun_stack.pop()
if not isinstance(tree, ParseNode) or tree.op.token != "~":
tree = ParseNode(unary_ops["~"], [tree], tree.origin)
return tree
