def annotated_tokens(code): prev_was_dot = False token_source = TokenSource(code) for (token_type, token) in token_source: if token_type == tokenize.ENDMARKER: break props = {} props["bare_ref"] = not prev_was_dot and token_type == tokenize.NAME props["bare_funcall"] = props["bare_ref"] and token_source.peek()[1] == "(" yield (token_type, token, props) prev_was_dot = token == "."
def annotated_tokens(code): prev_was_dot = False token_source = TokenSource(code) for (token_type, token) in token_source: if token_type == tokenize.ENDMARKER: break props = {} props["bare_ref"] = (not prev_was_dot and token_type == tokenize.NAME) props["bare_funcall"] = (props["bare_ref"] and token_source.peek()[1] == "(") yield (token_type, token, props) prev_was_dot = (token == ".")
def parse(code, extra_operators=[]): code = code.replace("\n", " ").strip() if not code: code = "~ 1" token_source = TokenSource(code) for extra_operator in extra_operators: if extra_operator.precedence < 0: raise ValueError, "all operators must have precedence >= 0" all_op_list = _default_ops + extra_operators unary_ops = {} binary_ops = {} for op in all_op_list: if op.arity == 1: unary_ops[op.token] = op elif op.arity == 2: binary_ops[op.token] = op else: raise ValueError, "operators must be unary or binary" c = _ParseContext(unary_ops, binary_ops) # This is an implementation of Dijkstra's shunting yard algorithm: # http://en.wikipedia.org/wiki/Shunting_yard_algorithm # http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm want_noun = True while True: if want_noun: want_noun = _read_noun_context(token_source, c) else: if token_source.peek()[0] == tokenize.ENDMARKER: break want_noun = _read_op_context(token_source, c) while c.op_stack: if c.op_stack[-1].token == "(": raise CharltonError("Unmatched '('", c.op_stack[-1]) _run_op(c) assert len(c.noun_stack) == 1 tree = c.noun_stack.pop() if not isinstance(tree, ParseNode) or tree.op.token != "~": tree = ParseNode(unary_ops["~"], [tree], tree.origin) return tree