def _read_python_expr(it, end_tokens):
# Read out a full python expression, stopping when we hit an
# unnested end token.
pytypes = []
token_strings = []
origins = []
bracket_level = 0
for pytype, token_string, origin in it:
assert bracket_level >= 0
if bracket_level == 0 and token_string in end_tokens:
it.push_back((pytype, token_string, origin))
break
if token_string in ("(", "[", "{"):
bracket_level += 1
if token_string in (")", "]", "}"):
bracket_level -= 1
if bracket_level < 0:
raise PatsyError("unmatched close bracket", origin)
pytypes.append(pytype)
token_strings.append(token_string)
origins.append(origin)
# Either we found an end_token, or we hit the end of the string
if bracket_level == 0:
expr_text = pretty_untokenize(zip(pytypes, token_strings))
if expr_text == "0":
token_type = "ZERO"
elif expr_text == "1":
token_type = "ONE"
elif _is_a(int, expr_text) or _is_a(float, expr_text):
token_type = "NUMBER"
else:
token_type = "PYTHON_EXPR"
return Token(token_type, Origin.combine(origins), extra=expr_text)
else:
raise PatsyError("unclosed bracket in embedded Python " "expression", Origin.combine(origins))
def _read_op_context(token, c):
if token.type == Token.RPAREN:
if c.trace:
print("Found close-paren")
while c.op_stack and c.op_stack[-1].op.token_type != Token.LPAREN:
_run_op(c)
if not c.op_stack:
raise PatsyError("missing '(' or extra ')'", token)
assert c.op_stack[-1].op.token_type == Token.LPAREN
# Expand the origin of the item on top of the noun stack to include
# the open and close parens:
combined = Origin.combine(
[c.op_stack[-1].token, c.noun_stack[-1].token, token])
c.noun_stack[-1].origin = combined
# Pop the open-paren
c.op_stack.pop()
return False
elif token.type in c.binary_ops:
if c.trace:
print("Found binary operator %r" % (token.type))
stackop = _StackOperator(c.binary_ops[token.type], token)
while (c.op_stack
and stackop.op.precedence <= c.op_stack[-1].op.precedence):
_run_op(c)
if c.trace:
print("Pushing binary operator %r" % (token.type))
c.op_stack.append(stackop)
return True
else:
raise PatsyError(
"expected an operator, not '%s'" %
(token.origin.relevant_code(), ), token)
def _read_op_context(token, c):
if token.type == Token.RPAREN:
if c.trace:
print "Found close-paren"
while c.op_stack and c.op_stack[-1].op.token_type != Token.LPAREN:
_run_op(c)
if not c.op_stack:
raise PatsyError("missing '(' or extra ')'", token)
assert c.op_stack[-1].op.token_type == Token.LPAREN
# Expand the origin of the item on top of the noun stack to include
# the open and close parens:
combined = Origin.combine([c.op_stack[-1].token,
c.noun_stack[-1].token,
token])
c.noun_stack[-1].origin = combined
# Pop the open-paren
c.op_stack.pop()
return False
elif token.type in c.binary_ops:
if c.trace:
print "Found binary operator %r" % (token.type)
stackop = _StackOperator(c.binary_ops[token.type], token)
while (c.op_stack
and stackop.op.precedence <= c.op_stack[-1].op.precedence):
_run_op(c)
if c.trace:
print "Pushing binary operator %r" % (token.type)
c.op_stack.append(stackop)
return True
else:
raise PatsyError("expected an operator, not '%s'"
% (token.origin.relevant_code(),),
token)
def _run_op(c):
assert c.op_stack
stackop = c.op_stack.pop()
args = []
for i in range(stackop.op.arity):
args.append(c.noun_stack.pop())
args.reverse()
if c.trace:
print("Reducing %r (%r)" % (stackop.op.token_type, args))
node = ParseNode(stackop.op.token_type, stackop.token, args,
Origin.combine([stackop.token] + args))
c.noun_stack.append(node)
def _run_op(c):
assert c.op_stack
stackop = c.op_stack.pop()
args = []
for i in xrange(stackop.op.arity):
args.append(c.noun_stack.pop())
args.reverse()
if c.trace:
print "Reducing %r (%r)" % (stackop.op.token_type, args)
node = ParseNode(stackop.op.token_type, stackop.token, args,
Origin.combine([stackop.token] + args))
c.noun_stack.append(node)
def _read_python_expr(it, end_tokens):
# Read out a full python expression, stopping when we hit an
# unnested end token.
pytypes = []
token_strings = []
origins = []
bracket_level = 0
for pytype, token_string, origin in it:
assert bracket_level >= 0
if bracket_level == 0 and token_string in end_tokens:
it.push_back((pytype, token_string, origin))
break
if token_string in ("(", "[", "{"):
bracket_level += 1
if token_string in (")", "]", "}"):
bracket_level -= 1
if bracket_level < 0:
raise PatsyError("unmatched close bracket", origin)
pytypes.append(pytype)
token_strings.append(token_string)
origins.append(origin)
# Either we found an end_token, or we hit the end of the string
if bracket_level == 0:
expr_text = pretty_untokenize(zip(pytypes, token_strings))
if expr_text == "0":
token_type = "ZERO"
elif expr_text == "1":
token_type = "ONE"
elif _is_a(int, expr_text) or _is_a(float, expr_text):
token_type = "NUMBER"
else:
token_type = "PYTHON_EXPR"
return Token(token_type, Origin.combine(origins), extra=expr_text)
else:
raise PatsyError("unclosed bracket in embedded Python "
"expression",
Origin.combine(origins))