def _Atom(self, children):
# type: (List[PNode]) -> expr_t
"""Handles alternatives of 'atom' where there is more than one child."""
tok = children[0].tok
id_ = tok.id
n = len(children)
if id_ == Id.Op_LParen:
# atom: '(' [yield_expr|testlist_comp] ')' | ...
if n == 2: # () is a tuple
assert children[1].tok.id == Id.Op_RParen, children[1]
return expr.Tuple([], expr_context_e.Store)
return self._TestlistComp(children[1], id_)
if id_ == Id.Op_LBracket:
# atom: ... | '[' [testlist_comp] ']' | ...
if n == 2: # []
assert children[1].tok.id == Id.Op_RBracket, children[1]
return expr.List([],
expr_context_e.Store) # unused expr_context_e
return self._TestlistComp(children[1], id_)
if id_ == Id.Op_LBrace:
return self._Dict(children[1])
if id_ == Id.Arith_Slash:
r = self._Regex(children[1])
flags = [] # type: List[token]
return expr.RegexLiteral(children[0].tok, r, flags)
raise NotImplementedError(id_)
def _Atom(self, children):
# type: (List[PNode]) -> expr_t
"""Handles alternatives of 'atom' where there is more than one child."""
tok = children[0].tok
id_ = tok.id
n = len(children)
if id_ == Id.Op_LParen:
# atom: '(' [yield_expr|testlist_comp] ')' | ...
if n == 2: # () is a tuple
assert children[1].tok.id == Id.Op_RParen, children[1]
return expr.Tuple([], expr_context_e.Store)
return self._TestlistComp(children[1], id_)
if id_ == Id.Op_LBracket:
# atom: ... | '[' [testlist_comp] ']' | ...
if n == 2: # []
assert children[1].tok.id == Id.Op_RBracket, children[1]
return expr.List([],
expr_context_e.Store) # unused expr_context_e
return self._TestlistComp(children[1], id_)
if id_ == Id.Op_LBrace:
# atom: ... | '{' [Op_Newline] [dict] '}'
i = 1
if children[i].tok.id == Id.Op_Newline:
i += 1
return self._Dict(children[i])
if id_ == Id.Arith_Slash:
r = self._Regex(children[1])
flags = [] # type: List[Token]
# TODO: Parse translation preference.
trans_pref = None # type: Token
return expr.RegexLiteral(children[0].tok, r, flags, trans_pref)
if id_ == Id.Expr_Func:
# STUB. This should really be a Func, not Lambda.
return expr.Lambda([], expr.Implicit())
raise NotImplementedError(Id_str(id_))
def _Atom(self, children):
# type: (List[PNode]) -> expr_t
"""Handles alternatives of 'atom' where there is more than one child."""
tok = children[0].tok
id_ = tok.id
if id_ == Id.Op_LParen:
# atom: '(' [yield_expr|testlist_comp] ')' | ...
if children[1].tok.id == Id.Op_RParen:
# () is a tuple
return expr.Tuple([], expr_context_e.Store)
else:
return self.Expr(children[1])
if id_ == Id.Op_LBracket:
# atom: ... | '[' [testlist_comp] ']' | ...
if len(children) == 2: # []
return expr.List([], expr_context_e.Store) # unused expr_context_e
p_list = children[1].children # what's between [ and ]
# [x for x in y]
if children[1].typ == grammar_nt.testlist_comp:
return self.Expr(children[1])
# [1, 2, 3]
n = len(p_list)
elts = []
for i in xrange(0, n, 2): # skip commas
p_node = p_list[i]
elts.append(self.Expr(p_node))
return expr.List(elts, expr_context_e.Store) # unused expr_context_e
if id_ == Id.Op_LBrace:
return self._Dict(children[1])
if id_ == Id.Arith_Slash:
r = self._Regex(children[1])
flags = [] # type: List[token]
return expr.RegexLiteral(children[0].tok, r, flags)
raise NotImplementedError(id_)
def Expr(self, pnode):
# type: (PNode) -> expr_t
"""Transform expressions (as opposed to statements)."""
typ = pnode.typ
tok = pnode.tok
children = pnode.children
if ISNONTERMINAL(typ):
c = '-' if not children else len(children)
#log('non-terminal %s %s', nt_name, c)
if typ == grammar_nt.oil_expr: # for if/while
# oil_expr: '(' testlist ')'
return self.Expr(children[1])
if typ == grammar_nt.return_expr: # for if/while
# return_expr: testlist end_stmt
return self.Expr(children[0])
if typ == grammar_nt.lvalue_list:
return self._AssocBinary(children)
if typ == grammar_nt.atom:
return self.atom(children)
if typ == grammar_nt.eval_input:
# testlist_input: testlist NEWLINE* ENDMARKER
return self.Expr(children[0])
if typ == grammar_nt.testlist:
# testlist: test (',' test)* [',']
return self._AssocBinary(children)
elif typ == grammar_nt.arith_expr:
# expr: term (('+'|'-') term)*
return self._AssocBinary(children)
elif typ == grammar_nt.term:
# term: factor (('*'|'/'|'div'|'mod') factor)*
return self._AssocBinary(children)
elif typ == grammar_nt.expr:
# expr: xor_expr ('|' xor_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.shift_expr:
# shift_expr: arith_expr (('<<'|'>>') arith_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.comparison:
# comparison: expr (comp_op expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.factor:
# factor: ('+'|'-'|'~') factor | power
# the power would have already been reduced
assert len(children) == 2, children
op, e = children
assert isinstance(op.tok, token)
return expr.Unary(op.tok, self.Expr(e))
elif typ == grammar_nt.atom_expr:
# atom_expr: ['await'] atom trailer*
# NOTE: This would be shorter in a recursive style.
base = self.Expr(children[0])
n = len(children)
for i in xrange(1, n):
pnode = children[i]
tok = pnode.tok
base = self.trailer(base, pnode)
return base
elif typ == grammar_nt.power:
# power: atom_expr ['^' factor]
# This doesn't repeat, so it doesn't matter if it's left or right
# associative.
return self._AssocBinary(children)
elif typ == grammar_nt.array_literal:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
items = [expr.Const(t) for t in tokens] # type: List[expr_t]
return expr.ArrayLiteral(left_tok, items)
elif typ == grammar_nt.sh_array_literal:
left_tok = children[0].tok
# HACK: When typ is Id.Expr_WordsDummy, the 'tok' field ('opaque')
# actually has a list of words!
typ1 = children[1].typ
assert typ1 == Id.Expr_WordsDummy.enum_id, typ1
array_words = cast('List[word_t]', children[1].tok)
return expr.ShellArrayLiteral(left_tok, array_words)
elif typ == grammar_nt.regex_literal:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Expr_Name
]
parts = [regex.Var(t) for t in tokens] # type: List[regex_t]
return expr.RegexLiteral(left_tok, regex.Concat(parts))
elif typ == grammar_nt.command_sub:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
words = [
word.Compound([word_part.Literal(t)]) for t in tokens
] # type: List[word_t]
return expr.CommandSub(left_tok, command.Simple(words))
elif typ == grammar_nt.sh_command_sub:
left_tok = children[0].tok
# HACK: When typ is Id.Expr_CommandDummy, the 'tok' field ('opaque')
# actually has a word_part.CommandSub!
typ1 = children[1].typ
assert typ1 == Id.Expr_CommandDummy.enum_id, typ1
cs_part = cast(word_part__CommandSub, children[1].tok)
# Awkward: the schemas are different
expr_part = expr.CommandSub(cs_part.left_token,
cs_part.command_list)
expr_part.spids.extend(cs_part.spids)
return expr_part
elif typ == grammar_nt.var_sub:
left_tok = children[0].tok
return expr.VarSub(left_tok, self.Expr(children[1]))
elif typ == grammar_nt.dq_string:
left_tok = children[0].tok
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
parts2 = [word_part.Literal(t)
for t in tokens] # type: List[word_part_t]
return expr.DoubleQuoted(left_tok, parts2)
else:
nt_name = self.number2symbol[typ]
raise AssertionError("PNode type %d (%s) wasn't handled" %
(typ, nt_name))
else: # Terminals should have a token
#log('terminal %s', tok)
if tok.id == Id.Expr_Name:
return expr.Var(tok)
elif tok.id == Id.Expr_Digits:
return expr.Const(tok)
else:
raise AssertionError(tok.id)
def Expr(self, pnode):
# type: (PNode) -> expr_t
"""Walk the homogeneous parse tree and create a typed AST."""
typ = pnode.typ
tok = pnode.tok
children = pnode.children
#if typ in self.number2symbol: # non-terminal
if ISNONTERMINAL(typ):
c = '-' if not children else len(children)
#log('non-terminal %s %s', nt_name, c)
if typ == grammar_nt.lvalue_list:
return self._AssocBinary(children)
if typ == grammar_nt.atom:
if children[0].tok.id == Id.Op_LParen:
return self.Expr(children[1])
else:
raise NotImplementedError
if typ == grammar_nt.eval_input:
# testlist_input: testlist NEWLINE* ENDMARKER
return self.Expr(children[0])
if typ == grammar_nt.testlist:
# testlist: test (',' test)* [',']
return self._AssocBinary(children)
elif typ == grammar_nt.arith_expr:
# expr: term (('+'|'-') term)*
return self._AssocBinary(children)
elif typ == grammar_nt.term:
# term: factor (('*'|'/'|'div'|'mod') factor)*
return self._AssocBinary(children)
elif typ == grammar_nt.expr:
# expr: xor_expr ('|' xor_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.shift_expr:
# shift_expr: arith_expr (('<<'|'>>') arith_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.comparison:
# comparison: expr (comp_op expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.factor:
# factor: ('+'|'-'|'~') factor | power
# the power would have already been reduced
assert len(children) == 2, children
op, e = children
assert isinstance(op.tok, syntax_asdl.token)
return expr.Unary(op.tok, self.Expr(e))
elif typ == grammar_nt.atom_expr:
# atom_expr: ['await'] atom trailer*
# NOTE: This would be shorter in a recursive style.
base = self.Expr(children[0])
n = len(children)
for i in xrange(1, n):
pnode = children[i]
tok = pnode.tok
base = self._Trailer(base, pnode)
return base
elif typ == grammar_nt.power:
# power: atom_expr ['^' factor]
# This doesn't repeat, so it doesn't matter if it's left or right
# associative.
return self._AssocBinary(children)
elif typ == grammar_nt.array_literal:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
array_words = [
word.CompoundWord([word_part.LiteralPart(t)])
for t in tokens
] # type: List[word_t]
return expr.ArrayLiteral(left_tok, array_words)
elif typ == grammar_nt.regex_literal:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Expr_Name
]
parts = [regex.Var(t) for t in tokens] # type: List[regex_t]
return expr.RegexLiteral(left_tok, regex.Concat(parts))
elif typ == grammar_nt.command_sub:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
words = [
word.CompoundWord([word_part.LiteralPart(t)])
for t in tokens
] # type: List[word_t]
return expr.CommandSub(left_tok, command.SimpleCommand(words))
elif typ == grammar_nt.expr_sub:
left_tok = children[0].tok
return expr.ExprSub(left_tok, self.Expr(children[1]))
elif typ == grammar_nt.var_sub:
left_tok = children[0].tok
return expr.VarSub(left_tok, self.Expr(children[1]))
elif typ == grammar_nt.dq_string:
left_tok = children[0].tok
tokens = [
pnode.tok for pnode in children[1:-1]
if pnode.tok.id == Id.Lit_Chars
]
parts2 = [oil_word_part.Literal(t)
for t in tokens] # type: List[oil_word_part_t]
return expr.DoubleQuoted(left_tok, parts2)
else:
nt_name = self.number2symbol[typ]
raise AssertionError("PNode type %d (%s) wasn't handled" %
(typ, nt_name))
else: # Terminals should have a token
#log('terminal %s', tok)
if tok.id == Id.Expr_Name:
return expr.Var(tok)
elif tok.id == Id.Expr_Digits:
return expr.Const(tok)
else:
raise AssertionError(tok.id)
