def Expr(self, pnode): # type: (PNode) -> expr_t """Transform expressions (as opposed to statements).""" typ = pnode.typ tok = pnode.tok children = pnode.children if ISNONTERMINAL(typ): # # Oil Entry Points / Additions # if typ == grammar_nt.oil_expr: # for if/while # oil_expr: '(' testlist ')' return self.Expr(children[1]) if typ == grammar_nt.return_expr: # return_expr: testlist end_stmt return self.Expr(children[0]) if typ == grammar_nt.place_list: return self._AssocBinary(children) if typ == grammar_nt.place: # place: NAME place_trailer* if len(pnode.children) == 1: return self.Expr(pnode.children[0]) # TODO: Called _Trailer but don't handle ( )? # only [] . -> :: ? raise NotImplementedError # # Python-like Expressions / Operators # if typ == grammar_nt.atom: if len(children) == 1: return self.Expr(children[0]) return self._Atom(children) if typ == grammar_nt.testlist: # testlist: test (',' test)* [','] # We need tuples for Python's 'var a, b = x' and 'for (a, b in x) {' return self._Tuple(children) if typ == grammar_nt.test: # test: or_test ['if' or_test 'else' test] | lambdef if len(children) == 1: return self.Expr(children[0]) # TODO: Handle lambdef test = self.Expr(children[2]) body = self.Expr(children[0]) orelse = self.Expr(children[4]) return expr.IfExp(test, body, orelse) if typ == grammar_nt.test_nocond: # test_nocond: or_test | lambdef_nocond assert len(children) == 1 return self.Expr(children[0]) if typ == grammar_nt.argument: # argument: ( test [comp_for] | # test '=' test | # '**' test | # '*' test ) if len(pnode.children) == 1: return self.Expr(children[0]) # TODO: raise NotImplementedError if typ == grammar_nt.subscript: # subscript: test | [test] ':' [test] [sliceop] if len(pnode.children) == 1: return self.Expr(children[0]) # TODO: raise NotImplementedError if typ == grammar_nt.testlist_comp: # testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) if children[1].typ == grammar_nt.comp_for: elt = self.Expr(children[0]) comp = self._CompFor(children[1]) return expr.ListComp(elt, [comp]) # (1,) (1, 2) etc. if children[1].tok.id == Id.Arith_Comma: return self._Tuple(children) raise NotImplementedError('testlist_comp') elif typ == grammar_nt.exprlist: # exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] if len(children) == 1: return self.Expr(children[0]) # used in for loop, genexpr. # TODO: This sould be placelist? for x, *y ? raise NotImplementedError('exprlist') # # Operators with Precedence # if typ == grammar_nt.or_test: # or_test: and_test ('or' and_test)* return self._AssocBinary(children) if typ == grammar_nt.and_test: # and_test: not_test ('and' not_test)* return self._AssocBinary(children) if typ == grammar_nt.not_test: # not_test: 'not' not_test | comparison if len(children) == 1: return self.Expr(children[0]) op_tok = children[0].tok # not return expr.Unary(op_tok, self.Expr(children[1])) elif typ == grammar_nt.comparison: if len(children) == 1: return self.Expr(children[0]) return self._CompareChain(children) elif typ == grammar_nt.expr: # expr: xor_expr ('|' xor_expr)* return self._AssocBinary(children) if typ == grammar_nt.xor_expr: # xor_expr: and_expr ('xor' and_expr)* return self._AssocBinary(children) if typ == grammar_nt.and_expr: # a & b # and_expr: shift_expr ('&' shift_expr)* return self._AssocBinary(children) elif typ == grammar_nt.shift_expr: # shift_expr: arith_expr (('<<'|'>>') arith_expr)* return self._AssocBinary(children) elif typ == grammar_nt.arith_expr: # arith_expr: term (('+'|'-') term)* return self._AssocBinary(children) elif typ == grammar_nt.term: # term: factor (('*'|'/'|'div'|'mod') factor)* return self._AssocBinary(children) elif typ == grammar_nt.factor: # factor: ('+'|'-'|'~') factor | power # the power would have already been reduced if len(children) == 1: return self.Expr(children[0]) op, e = children assert isinstance(op.tok, token) return expr.Unary(op.tok, self.Expr(e)) elif typ == grammar_nt.power: # power: atom trailer* ['^' factor] node = self.Expr(children[0]) if len(children) == 1: # No trailers return node n = len(children) i = 1 while i < n and ISNONTERMINAL(children[i].typ): node = self._Trailer(node, children[i]) i += 1 if i != n: # ['^' factor] op_tok = children[i].tok assert op_tok.id == Id.Arith_Caret, op_tok factor = self.Expr(children[i+1]) node = expr.Binary(op_tok, node, factor) return node # # Oil Lexer Modes # elif typ == grammar_nt.array_literal: left_tok = children[0].tok # Approximation for now. tokens = [ pnode.tok for pnode in children[1:-1] if pnode.tok.id == Id.Lit_Chars ] items = [expr.Const(t) for t in tokens] # type: List[expr_t] return expr.ArrayLiteral(left_tok, items) elif typ == grammar_nt.sh_array_literal: left_tok = children[0].tok # HACK: When typ is Id.Expr_CastedDummy, the 'tok' field ('opaque') # actually has a list of words! typ1 = children[1].typ assert typ1 == Id.Expr_CastedDummy.enum_id, typ1 array_words = cast('List[word_t]', children[1].tok) return sh_array_literal(left_tok, array_words) elif typ == grammar_nt.sh_command_sub: return cast(command_sub, children[1].tok) elif typ == grammar_nt.braced_var_sub: return cast(braced_var_sub, children[1].tok) elif typ == grammar_nt.dq_string: return cast(double_quoted, children[1].tok) elif typ == grammar_nt.sq_string: return cast(single_quoted, children[1].tok) elif typ == grammar_nt.simple_var_sub: return simple_var_sub(children[0].tok) else: nt_name = self.number2symbol[typ] raise AssertionError( "PNode type %d (%s) wasn't handled" % (typ, nt_name)) else: # Terminals should have a token id_ = tok.id if id_ == Id.Expr_Name: return expr.Var(tok) if id_ in ( Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt, Id.Expr_HexInt, Id.Expr_Float): return expr.Const(tok) if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False): return expr.Const(tok) from core.meta import IdInstance raise NotImplementedError(IdInstance(typ))
def _PushOilTokens(parse_ctx, gr, p, lex): # type: (ParseContext, Grammar, parse.Parser, Lexer) -> token """Push tokens onto pgen2's parser. Returns the last token so it can be reused/seen by the CommandParser. """ #log('keywords = %s', gr.keywords) #log('tokens = %s', gr.tokens) last_token = None # type: Optional[token] balance = 0 # to ignore newlines while True: if last_token: # e.g. left over from WordParser tok = last_token #log('last_token = %s', last_token) last_token = None else: tok = lex.Read(lex_mode_e.Expr) #log('tok = %s', tok) # Comments and whitespace. Newlines aren't ignored. if lookup.LookupKind(tok.id) == Kind.Ignored: continue # For var x = { # a: 1, b: 2 # } if balance > 0 and tok.id == Id.Op_Newline: #log('*** SKIPPING NEWLINE') continue balance += _OTHER_BALANCE.get(tok.id, 0) #log('BALANCE after seeing %s = %d', tok.id, balance) #if tok.id == Id.Expr_Name and tok.val in KEYWORDS: # tok.id = KEYWORDS[tok.val] # log('Replaced with %s', tok.id) assert tok.id < 256, Id_str(tok.id) ilabel = _Classify(gr, tok) #log('tok = %s, ilabel = %d', tok, ilabel) if p.addtoken(tok.id, tok, ilabel): return tok # # Mututally recursive calls into the command/word parsers. # if mylib.PYTHON: if tok.id == Id.Left_AtParen: left_tok = tok lex.PushHint(Id.Op_RParen, Id.Right_ShArrayLiteral) # Blame the opening token line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if 0: log('w = %s', w) if isinstance(w, word__Token): word_id = word_.CommandId(w) if word_id == Id.Right_ShArrayLiteral: break elif word_id == Id.Op_Newline: # internal newlines allowed continue else: # Token p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__Compound) # for MyPy words.append(w) words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) typ = Id.Expr_CastedDummy lit_part = sh_array_literal(left_tok, words3) opaque = cast(token, lit_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) tok = w.token ilabel = _Classify(gr, tok) done = p.addtoken(tok.id, tok, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DollarParen: left_token = tok lex.PushHint(Id.Op_RParen, Id.Eof_RParen) line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) c_parser = parse_ctx.MakeParserForCommandSub( line_reader, lex, Id.Eof_RParen) node = c_parser.ParseCommandSub() # A little gross: Copied from osh/word_parse.py right_token = c_parser.w_parser.cur_token cs_part = command_sub(left_token, node) cs_part.spids.append(left_token.span_id) cs_part.spids.append(right_token.span_id) typ = Id.Expr_CastedDummy opaque = cast(token, cs_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) ilabel = _Classify(gr, right_token) done = p.addtoken(right_token.id, right_token, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DoubleQuote: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) parts = [] # type: List[word_part_t] last_token = w_parser.ReadDoubleQuoted(left_token, parts) expr_dq_part = double_quoted(left_token, parts) typ = Id.Expr_CastedDummy opaque = cast(token, expr_dq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue if tok.id == Id.Left_DollarBrace: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) part, last_token = w_parser.ReadBracedBracedVarSub(left_token) # It's casted word_part__BracedVarSub -> dummy -> expr__BracedVarSub! typ = Id.Expr_CastedDummy opaque = cast(token, part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue # '' and c'' if tok.id in (Id.Left_SingleQuoteRaw, Id.Left_SingleQuoteC): if tok.id == Id.Left_SingleQuoteRaw: sq_mode = lex_mode_e.SQ_Raw else: sq_mode = lex_mode_e.SQ_C left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) tokens = [] # type: List[token] no_backslashes = (left_token.val == "'") last_token = w_parser.ReadSingleQuoted(sq_mode, left_token, tokens, no_backslashes) sq_part = single_quoted(left_token, tokens) typ = Id.Expr_CastedDummy opaque = cast(token, sq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue else: # We never broke out -- EOF is too soon (how can this happen???) raise parse.ParseError("incomplete input", tok.id, tok)
def _ReadArrayLiteral(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =', token=self.cur_token) left_token = self.cur_token paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = self.parse_ctx.MakeWordParser(self.lexer, self.line_reader) words = [] # type: List[compound_word] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if w.tag_() == word_e.Token: tok = cast(Token, w) if tok.id == Id.Right_ShArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif tok.id == Id.Op_Newline: continue else: # Token p_die('Unexpected token in array literal', word=w) words.append(cast(compound_word, w)) if len(words) == 0: # a=() is empty indexed array # Needed for type safety, doh no_words = [] # type: List[word_t] node = sh_array_literal(left_token, no_words) node.spids.append(left_token.span_id) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word_.DetectAssocPair(words[0]) if pair: k, v = pair pairs = [k, v] n = len(words) for i in xrange(1, n): w2 = words[i] pair = word_.DetectAssocPair(w2) if not pair: p_die("Expected associative array pair", word=w2) k, v = pair pairs.append(k) # flat representation pairs.append(v) # invariant List? node2 = word_part.AssocArrayLiteral(left_token, pairs) node2.spids.append(paren_spid) return node2 # Brace detection for arrays but NOT associative arrays words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) node = sh_array_literal(left_token, words3) node.spids.append(paren_spid) return node
def Expr(self, pnode): # type: (PNode) -> expr_t """Transform expressions (as opposed to statements).""" typ = pnode.typ tok = pnode.tok children = pnode.children if ISNONTERMINAL(typ): # # Oil Entry Points / Additions # if typ == grammar_nt.oil_expr: # for if/while # oil_expr: '(' testlist ')' return self.Expr(children[1]) if typ == grammar_nt.command_expr: # return_expr: testlist end_stmt return self.Expr(children[0]) # # Python-like Expressions / Operators # if typ == grammar_nt.atom: if len(children) == 1: return self.Expr(children[0]) return self._Atom(children) if typ == grammar_nt.testlist: # testlist: test (',' test)* [','] return self._Tuple(children) if typ == grammar_nt.test: # test: or_test ['if' or_test 'else' test] | lambdef if len(children) == 1: return self.Expr(children[0]) # TODO: Handle lambdef test = self.Expr(children[2]) body = self.Expr(children[0]) orelse = self.Expr(children[4]) return expr.IfExp(test, body, orelse) if typ == grammar_nt.lambdef: # lambdef: '|' [name_type_list] '|' test n = len(children) if n == 4: params = self._NameTypeList(children[1]) else: params = [] body = self.Expr(children[n - 1]) return expr.Lambda(params, body) # # Operators with Precedence # if typ == grammar_nt.or_test: # or_test: and_test ('or' and_test)* return self._AssocBinary(children) if typ == grammar_nt.and_test: # and_test: not_test ('and' not_test)* return self._AssocBinary(children) if typ == grammar_nt.not_test: # not_test: 'not' not_test | comparison if len(children) == 1: return self.Expr(children[0]) op_tok = children[0].tok # not return expr.Unary(op_tok, self.Expr(children[1])) elif typ == grammar_nt.comparison: if len(children) == 1: return self.Expr(children[0]) return self._CompareChain(children) elif typ == grammar_nt.range_expr: if len(children) == 1: return self.Expr(children[0]) if len(children) == 3: return expr.Range(self.Expr(children[0]), self.Expr(children[2])) raise AssertionError(children) elif typ == grammar_nt.expr: # expr: xor_expr ('|' xor_expr)* return self._AssocBinary(children) if typ == grammar_nt.xor_expr: # xor_expr: and_expr ('xor' and_expr)* return self._AssocBinary(children) if typ == grammar_nt.and_expr: # a & b # and_expr: shift_expr ('&' shift_expr)* return self._AssocBinary(children) elif typ == grammar_nt.shift_expr: # shift_expr: arith_expr (('<<'|'>>') arith_expr)* return self._AssocBinary(children) elif typ == grammar_nt.arith_expr: # arith_expr: term (('+'|'-') term)* return self._AssocBinary(children) elif typ == grammar_nt.term: # term: factor (('*'|'/'|'div'|'mod') factor)* return self._AssocBinary(children) elif typ == grammar_nt.factor: # factor: ('+'|'-'|'~') factor | power # the power would have already been reduced if len(children) == 1: return self.Expr(children[0]) op, e = children assert isinstance(op.tok, token) return expr.Unary(op.tok, self.Expr(e)) elif typ == grammar_nt.power: # power: atom trailer* ['^' factor] node = self.Expr(children[0]) if len(children) == 1: # No trailers return node n = len(children) i = 1 while i < n and ISNONTERMINAL(children[i].typ): node = self._Trailer(node, children[i]) i += 1 if i != n: # ['^' factor] op_tok = children[i].tok assert op_tok.id == Id.Arith_Caret, op_tok factor = self.Expr(children[i + 1]) node = expr.Binary(op_tok, node, factor) return node elif typ == grammar_nt.array_literal: left_tok = children[0].tok items = [self._ArrayItem(p) for p in children[1:-1]] return expr.ArrayLiteral(left_tok, items) elif typ == grammar_nt.oil_expr_sub: return self.Expr(children[0]) # # Oil Lexer Modes # elif typ == grammar_nt.sh_array_literal: left_tok = children[0].tok # HACK: When typ is Id.Expr_CastedDummy, the 'tok' field ('opaque') # actually has a list of words! typ1 = children[1].typ assert typ1 == Id.Expr_CastedDummy.enum_id, typ1 array_words = cast('List[word_t]', children[1].tok) return sh_array_literal(left_tok, array_words) elif typ == grammar_nt.sh_command_sub: return cast(command_sub, children[1].tok) elif typ == grammar_nt.braced_var_sub: return cast(braced_var_sub, children[1].tok) elif typ == grammar_nt.dq_string: return cast(double_quoted, children[1].tok) elif typ == grammar_nt.sq_string: return cast(single_quoted, children[1].tok) elif typ == grammar_nt.simple_var_sub: return simple_var_sub(children[0].tok) else: nt_name = self.number2symbol[typ] raise AssertionError("PNode type %d (%s) wasn't handled" % (typ, nt_name)) else: # Terminals should have a token id_ = tok.id if id_ == Id.Expr_Name: return expr.Var(tok) if id_ in (Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt, Id.Expr_HexInt, Id.Expr_Float): return expr.Const(tok) if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False): return expr.Const(tok) from core.meta import IdInstance raise NotImplementedError(IdInstance(typ))
def _ReadArrayLiteral(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) left_token = self.cur_token paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = self.parse_ctx.MakeWordParser(self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if isinstance(w, word__Token): word_id = word_.CommandId(w) if word_id == Id.Right_ShArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # Token p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__Compound) # for MyPy words.append(w) if not words: # a=() is empty indexed array # ignore for invariant List? node = sh_array_literal(left_token, words) # type: ignore node.spids.append(left_token.span_id) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word_.DetectAssocPair(words[0]) if pair: pairs = [pair[0], pair[1]] # flat representation n = len(words) for i in xrange(1, n): w = words[i] pair = word_.DetectAssocPair(w) if not pair: p_die("Expected associative array pair", word=w) pairs.append(pair[0]) # flat representation pairs.append(pair[1]) # invariant List? node = word_part.AssocArrayLiteral(left_token, pairs) # type: ignore node.spids.append(paren_spid) return node words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) node = sh_array_literal(left_token, words3) node.spids.append(paren_spid) return node