def _Atom(self, children): # type: (List[PNode]) -> expr_t """Handles alternatives of 'atom' where there is more than one child.""" tok = children[0].tok id_ = tok.id n = len(children) if id_ == Id.Op_LParen: # atom: '(' [yield_expr|testlist_comp] ')' | ... if n == 2: # () is a tuple assert children[1].tok.id == Id.Op_RParen, children[1] return expr.Tuple([], expr_context_e.Store) return self._TestlistComp(children[1], id_) if id_ == Id.Op_LBracket: # atom: ... | '[' [testlist_comp] ']' | ... if n == 2: # [] assert children[1].tok.id == Id.Op_RBracket, children[1] return expr.List([], expr_context_e.Store) # unused expr_context_e return self._TestlistComp(children[1], id_) if id_ == Id.Op_LBrace: return self._Dict(children[1]) if id_ == Id.Arith_Slash: r = self._Regex(children[1]) flags = [] # type: List[token] return expr.RegexLiteral(children[0].tok, r, flags) raise NotImplementedError(id_)
def _Atom(self, children): # type: (List[PNode]) -> expr_t """Handles alternatives of 'atom' where there is more than one child.""" tok = children[0].tok id_ = tok.id n = len(children) if id_ == Id.Op_LParen: # atom: '(' [yield_expr|testlist_comp] ')' | ... if n == 2: # () is a tuple assert children[1].tok.id == Id.Op_RParen, children[1] return expr.Tuple([], expr_context_e.Store) return self._TestlistComp(children[1], id_) if id_ == Id.Op_LBracket: # atom: ... | '[' [testlist_comp] ']' | ... if n == 2: # [] assert children[1].tok.id == Id.Op_RBracket, children[1] return expr.List([], expr_context_e.Store) # unused expr_context_e return self._TestlistComp(children[1], id_) if id_ == Id.Op_LBrace: # atom: ... | '{' [Op_Newline] [dict] '}' i = 1 if children[i].tok.id == Id.Op_Newline: i += 1 return self._Dict(children[i]) if id_ == Id.Arith_Slash: r = self._Regex(children[1]) flags = [] # type: List[Token] # TODO: Parse translation preference. trans_pref = None # type: Token return expr.RegexLiteral(children[0].tok, r, flags, trans_pref) if id_ == Id.Expr_Func: # STUB. This should really be a Func, not Lambda. return expr.Lambda([], expr.Implicit()) raise NotImplementedError(Id_str(id_))
def _Atom(self, children): # type: (List[PNode]) -> expr_t """Handles alternatives of 'atom' where there is more than one child.""" tok = children[0].tok id_ = tok.id if id_ == Id.Op_LParen: # atom: '(' [yield_expr|testlist_comp] ')' | ... if children[1].tok.id == Id.Op_RParen: # () is a tuple return expr.Tuple([], expr_context_e.Store) else: return self.Expr(children[1]) if id_ == Id.Op_LBracket: # atom: ... | '[' [testlist_comp] ']' | ... if len(children) == 2: # [] return expr.List([], expr_context_e.Store) # unused expr_context_e p_list = children[1].children # what's between [ and ] # [x for x in y] if children[1].typ == grammar_nt.testlist_comp: return self.Expr(children[1]) # [1, 2, 3] n = len(p_list) elts = [] for i in xrange(0, n, 2): # skip commas p_node = p_list[i] elts.append(self.Expr(p_node)) return expr.List(elts, expr_context_e.Store) # unused expr_context_e if id_ == Id.Op_LBrace: return self._Dict(children[1]) if id_ == Id.Arith_Slash: r = self._Regex(children[1]) flags = [] # type: List[token] return expr.RegexLiteral(children[0].tok, r, flags) raise NotImplementedError(id_)
def Expr(self, pnode): # type: (PNode) -> expr_t """Transform expressions (as opposed to statements).""" typ = pnode.typ tok = pnode.tok children = pnode.children if ISNONTERMINAL(typ): c = '-' if not children else len(children) #log('non-terminal %s %s', nt_name, c) if typ == grammar_nt.oil_expr: # for if/while # oil_expr: '(' testlist ')' return self.Expr(children[1]) if typ == grammar_nt.return_expr: # for if/while # return_expr: testlist end_stmt return self.Expr(children[0]) if typ == grammar_nt.lvalue_list: return self._AssocBinary(children) if typ == grammar_nt.atom: return self.atom(children) if typ == grammar_nt.eval_input: # testlist_input: testlist NEWLINE* ENDMARKER return self.Expr(children[0]) if typ == grammar_nt.testlist: # testlist: test (',' test)* [','] return self._AssocBinary(children) elif typ == grammar_nt.arith_expr: # expr: term (('+'|'-') term)* return self._AssocBinary(children) elif typ == grammar_nt.term: # term: factor (('*'|'/'|'div'|'mod') factor)* return self._AssocBinary(children) elif typ == grammar_nt.expr: # expr: xor_expr ('|' xor_expr)* return self._AssocBinary(children) elif typ == grammar_nt.shift_expr: # shift_expr: arith_expr (('<<'|'>>') arith_expr)* return self._AssocBinary(children) elif typ == grammar_nt.comparison: # comparison: expr (comp_op expr)* return self._AssocBinary(children) elif typ == grammar_nt.factor: # factor: ('+'|'-'|'~') factor | power # the power would have already been reduced assert len(children) == 2, children op, e = children assert isinstance(op.tok, token) return expr.Unary(op.tok, self.Expr(e)) elif typ == grammar_nt.atom_expr: # atom_expr: ['await'] atom trailer* # NOTE: This would be shorter in a recursive style. base = self.Expr(children[0]) n = len(children) for i in xrange(1, n): pnode = children[i] tok = pnode.tok base = self.trailer(base, pnode) return base elif typ == grammar_nt.power: # power: atom_expr ['^' factor] # This doesn't repeat, so it doesn't matter if it's left or right # associative. return self._AssocBinary(children) elif typ == grammar_nt.array_literal: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] items = [expr.Const(t) for t in tokens] # type: List[expr_t] return expr.ArrayLiteral(left_tok, items) elif typ == grammar_nt.sh_array_literal: left_tok = children[0].tok # HACK: When typ is Id.Expr_WordsDummy, the 'tok' field ('opaque') # actually has a list of words! typ1 = children[1].typ assert typ1 == Id.Expr_WordsDummy.enum_id, typ1 array_words = cast('List[word_t]', children[1].tok) return expr.ShellArrayLiteral(left_tok, array_words) elif typ == grammar_nt.regex_literal: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Expr_Name ] parts = [regex.Var(t) for t in tokens] # type: List[regex_t] return expr.RegexLiteral(left_tok, regex.Concat(parts)) elif typ == grammar_nt.command_sub: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] words = [ word.Compound([word_part.Literal(t)]) for t in tokens ] # type: List[word_t] return expr.CommandSub(left_tok, command.Simple(words)) elif typ == grammar_nt.sh_command_sub: left_tok = children[0].tok # HACK: When typ is Id.Expr_CommandDummy, the 'tok' field ('opaque') # actually has a word_part.CommandSub! typ1 = children[1].typ assert typ1 == Id.Expr_CommandDummy.enum_id, typ1 cs_part = cast(word_part__CommandSub, children[1].tok) # Awkward: the schemas are different expr_part = expr.CommandSub(cs_part.left_token, cs_part.command_list) expr_part.spids.extend(cs_part.spids) return expr_part elif typ == grammar_nt.var_sub: left_tok = children[0].tok return expr.VarSub(left_tok, self.Expr(children[1])) elif typ == grammar_nt.dq_string: left_tok = children[0].tok tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] parts2 = [word_part.Literal(t) for t in tokens] # type: List[word_part_t] return expr.DoubleQuoted(left_tok, parts2) else: nt_name = self.number2symbol[typ] raise AssertionError("PNode type %d (%s) wasn't handled" % (typ, nt_name)) else: # Terminals should have a token #log('terminal %s', tok) if tok.id == Id.Expr_Name: return expr.Var(tok) elif tok.id == Id.Expr_Digits: return expr.Const(tok) else: raise AssertionError(tok.id)
def Expr(self, pnode): # type: (PNode) -> expr_t """Walk the homogeneous parse tree and create a typed AST.""" typ = pnode.typ tok = pnode.tok children = pnode.children #if typ in self.number2symbol: # non-terminal if ISNONTERMINAL(typ): c = '-' if not children else len(children) #log('non-terminal %s %s', nt_name, c) if typ == grammar_nt.lvalue_list: return self._AssocBinary(children) if typ == grammar_nt.atom: if children[0].tok.id == Id.Op_LParen: return self.Expr(children[1]) else: raise NotImplementedError if typ == grammar_nt.eval_input: # testlist_input: testlist NEWLINE* ENDMARKER return self.Expr(children[0]) if typ == grammar_nt.testlist: # testlist: test (',' test)* [','] return self._AssocBinary(children) elif typ == grammar_nt.arith_expr: # expr: term (('+'|'-') term)* return self._AssocBinary(children) elif typ == grammar_nt.term: # term: factor (('*'|'/'|'div'|'mod') factor)* return self._AssocBinary(children) elif typ == grammar_nt.expr: # expr: xor_expr ('|' xor_expr)* return self._AssocBinary(children) elif typ == grammar_nt.shift_expr: # shift_expr: arith_expr (('<<'|'>>') arith_expr)* return self._AssocBinary(children) elif typ == grammar_nt.comparison: # comparison: expr (comp_op expr)* return self._AssocBinary(children) elif typ == grammar_nt.factor: # factor: ('+'|'-'|'~') factor | power # the power would have already been reduced assert len(children) == 2, children op, e = children assert isinstance(op.tok, syntax_asdl.token) return expr.Unary(op.tok, self.Expr(e)) elif typ == grammar_nt.atom_expr: # atom_expr: ['await'] atom trailer* # NOTE: This would be shorter in a recursive style. base = self.Expr(children[0]) n = len(children) for i in xrange(1, n): pnode = children[i] tok = pnode.tok base = self._Trailer(base, pnode) return base elif typ == grammar_nt.power: # power: atom_expr ['^' factor] # This doesn't repeat, so it doesn't matter if it's left or right # associative. return self._AssocBinary(children) elif typ == grammar_nt.array_literal: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] array_words = [ word.CompoundWord([word_part.LiteralPart(t)]) for t in tokens ] # type: List[word_t] return expr.ArrayLiteral(left_tok, array_words) elif typ == grammar_nt.regex_literal: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Expr_Name ] parts = [regex.Var(t) for t in tokens] # type: List[regex_t] return expr.RegexLiteral(left_tok, regex.Concat(parts)) elif typ == grammar_nt.command_sub: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] words = [ word.CompoundWord([word_part.LiteralPart(t)]) for t in tokens ] # type: List[word_t] return expr.CommandSub(left_tok, command.SimpleCommand(words)) elif typ == grammar_nt.expr_sub: left_tok = children[0].tok return expr.ExprSub(left_tok, self.Expr(children[1])) elif typ == grammar_nt.var_sub: left_tok = children[0].tok return expr.VarSub(left_tok, self.Expr(children[1])) elif typ == grammar_nt.dq_string: left_tok = children[0].tok tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] parts2 = [oil_word_part.Literal(t) for t in tokens] # type: List[oil_word_part_t] return expr.DoubleQuoted(left_tok, parts2) else: nt_name = self.number2symbol[typ] raise AssertionError("PNode type %d (%s) wasn't handled" % (typ, nt_name)) else: # Terminals should have a token #log('terminal %s', tok) if tok.id == Id.Expr_Name: return expr.Var(tok) elif tok.id == Id.Expr_Digits: return expr.Const(tok) else: raise AssertionError(tok.id)