def _MakeSimpleCommand(preparsed_list, suffix_words, redirects): """Create an command.SimpleCommand node.""" # FOO=(1 2 3) ls is not allowed. for _, _, _, w in preparsed_list: if word.HasArrayPart(w): p_die("Environment bindings can't contain array literals", word=w) # echo FOO=(1 2 3) is not allowed (but we should NOT fail on echo FOO[x]=1). for w in suffix_words: if word.HasArrayPart(w): p_die("Commands can't contain array literals", word=w) # NOTE: # In bash, {~bob,~jane}/src works, even though ~ isn't the leading # character of the initial word. # However, this means we must do tilde detection AFTER brace EXPANSION, not # just after brace DETECTION like we're doing here. # The BracedWordTree instances have to be expanded into CompoundWord # instances for the tilde detection to work. words2 = braces.BraceDetectAll(suffix_words) words3 = word.TildeDetectAll(words2) node = command.SimpleCommand() node.words = words3 node.redirects = redirects _AppendMoreEnv(preparsed_list, node.more_env) return node
def _ReadArrayLiteralPart(self): self._Next(lex_mode_e.Outer) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None if w.tag == word_e.TokenWord: word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) words.append(w) words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) return word_part.ArrayLiteralPart(words3)
def _ParseForEachLoop(self): node = command.ForEach() node.do_arg_iter = False ok, iter_name, quoted = word.StaticEval(self.cur_word) if not ok or quoted: p_die("Loop variable name should be a constant", word=self.cur_word) if not match.IsValidVarName(iter_name): p_die("Invalid loop variable name", word=self.cur_word) node.iter_name = iter_name self._Next() # skip past name self._NewlineOk() in_spid = const.NO_INTEGER semi_spid = const.NO_INTEGER self._Peek() if self.c_id == Id.KW_In: self._Next() # skip in in_spid = word.LeftMostSpanForWord(self.cur_word) + 1 iter_words, semi_spid = self.ParseForWords() assert iter_words is not None words2 = braces.BraceDetectAll(iter_words) words3 = word.TildeDetectAll(words2) node.iter_words = words3 elif self.c_id == Id.Op_Semi: node.do_arg_iter = True # implicit for loop self._Next() elif self.c_id == Id.KW_Do: node.do_arg_iter = True # implicit for loop # do not advance else: # for foo BAD p_die('Unexpected word after for loop variable', word=self.cur_word) node.spids.extend((in_spid, semi_spid)) body_node = self.ParseDoGroup() assert body_node is not None node.body = body_node return node
def _PushOilTokens(parse_ctx, gr, p, lex): # type: (ParseContext, Grammar, parse.Parser, Lexer) -> token """Push tokens onto pgen2's parser. Returns the last token so it can be reused/seen by the CommandParser. """ #log('keywords = %s', gr.keywords) #log('tokens = %s', gr.tokens) mode = lex_mode_e.Expr mode_stack = [mode] last_token = None balance = 0 from core.util import log while True: if last_token: # e.g. left over from WordParser tok = last_token #log('last_token = %s', last_token) last_token = None else: tok = lex.Read(mode) #log('tok = %s', tok) # Comments and whitespace. Newlines aren't ignored. if meta.LookupKind(tok.id) == Kind.Ignored: continue # For var x = { # a: 1, b: 2 # } if balance > 0 and tok.id == Id.Op_Newline: #log('*** SKIPPING NEWLINE') continue action = _MODE_TRANSITIONS.get((mode, tok.id)) if action == POP: mode_stack.pop() mode = mode_stack[-1] balance -= 1 #log('POPPED to %s', mode) elif action: # it's an Id new_mode = action mode_stack.append(new_mode) mode = new_mode balance += 1 # e.g. var x = $/ NEWLINE / #log('PUSHED to %s', mode) else: # If we didn't already so something with the balance, look at another table. balance += _OTHER_BALANCE.get(tok.id, 0) #log('BALANCE after seeing %s = %d', tok.id, balance) #if tok.id == Id.Expr_Name and tok.val in KEYWORDS: # tok.id = KEYWORDS[tok.val] # log('Replaced with %s', tok.id) if tok.id.enum_id >= 256: raise AssertionError(str(tok)) ilabel = _Classify(gr, tok) #log('tok = %s, ilabel = %d', tok, ilabel) if p.addtoken(tok.id.enum_id, tok, ilabel): return tok # # Extra handling of the body of @() and $(). Lex in the ShCommand mode. # if tok.id == Id.Left_AtParen: lex.PushHint(Id.Op_RParen, Id.Right_ShArrayLiteral) # Blame the opening token line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if 0: log('w = %s', w) if isinstance(w, word__Token): word_id = word_.CommandId(w) if word_id == Id.Right_ShArrayLiteral: break elif word_id == Id.Op_Newline: # internal newlines allowed continue else: # Token p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__Compound) # for MyPy words.append(w) words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) typ = Id.Expr_CastedDummy.enum_id opaque = cast(token, words3) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) tok = w.token ilabel = _Classify(gr, tok) done = p.addtoken(tok.id.enum_id, tok, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DollarParen: left_token = tok lex.PushHint(Id.Op_RParen, Id.Eof_RParen) line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) c_parser = parse_ctx.MakeParserForCommandSub(line_reader, lex, Id.Eof_RParen) node = c_parser.ParseCommandSub() # A little gross: Copied from osh/word_parse.py right_token = c_parser.w_parser.cur_token cs_part = command_sub(left_token, node) cs_part.spids.append(left_token.span_id) cs_part.spids.append(right_token.span_id) typ = Id.Expr_CastedDummy.enum_id opaque = cast(token, cs_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) ilabel = _Classify(gr, right_token) done = p.addtoken(right_token.id.enum_id, right_token, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DoubleQuote: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) parts = [] # type: List[word_part_t] last_token = w_parser.ReadDoubleQuoted(left_token, parts) expr_dq_part = double_quoted(left_token, parts) typ = Id.Expr_CastedDummy.enum_id opaque = cast(token, expr_dq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue if tok.id == Id.Left_DollarBrace: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) part, last_token = w_parser.ReadBracedBracedVarSub(left_token) # It's casted word_part__BracedVarSub -> dummy -> expr__BracedVarSub! typ = Id.Expr_CastedDummy.enum_id opaque = cast(token, part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue # '' and c'' if tok.id in (Id.Left_SingleQuoteRaw, Id.Left_SingleQuoteC): left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) # mode can be SQ or DollarSQ tokens = [] # type: List[token] no_backslashes = (left_token.val == "'") last_token = w_parser.ReadSingleQuoted(mode, left_token, tokens, no_backslashes) sq_part = single_quoted(left_token, tokens) typ = Id.Expr_CastedDummy.enum_id opaque = cast(token, sq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue else: # We never broke out -- EOF is too soon (how can this happen???) raise parse.ParseError("incomplete input", tok.id.enum_id, tok)
def _PushOilTokens(parse_ctx, gr, p, lex): # type: (ParseContext, Grammar, parse.Parser, Lexer) -> token """Push tokens onto pgen2's parser. Returns the last token so it can be reused/seen by the CommandParser. """ #log('keywords = %s', gr.keywords) #log('tokens = %s', gr.tokens) last_token = None # type: Optional[token] balance = 0 # to ignore newlines while True: if last_token: # e.g. left over from WordParser tok = last_token #log('last_token = %s', last_token) last_token = None else: tok = lex.Read(lex_mode_e.Expr) #log('tok = %s', tok) # Comments and whitespace. Newlines aren't ignored. if lookup.LookupKind(tok.id) == Kind.Ignored: continue # For var x = { # a: 1, b: 2 # } if balance > 0 and tok.id == Id.Op_Newline: #log('*** SKIPPING NEWLINE') continue balance += _OTHER_BALANCE.get(tok.id, 0) #log('BALANCE after seeing %s = %d', tok.id, balance) #if tok.id == Id.Expr_Name and tok.val in KEYWORDS: # tok.id = KEYWORDS[tok.val] # log('Replaced with %s', tok.id) assert tok.id < 256, Id_str(tok.id) ilabel = _Classify(gr, tok) #log('tok = %s, ilabel = %d', tok, ilabel) if p.addtoken(tok.id, tok, ilabel): return tok # # Mututally recursive calls into the command/word parsers. # if mylib.PYTHON: if tok.id == Id.Left_AtParen: left_tok = tok lex.PushHint(Id.Op_RParen, Id.Right_ShArrayLiteral) # Blame the opening token line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if 0: log('w = %s', w) if isinstance(w, word__Token): word_id = word_.CommandId(w) if word_id == Id.Right_ShArrayLiteral: break elif word_id == Id.Op_Newline: # internal newlines allowed continue else: # Token p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__Compound) # for MyPy words.append(w) words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) typ = Id.Expr_CastedDummy lit_part = sh_array_literal(left_tok, words3) opaque = cast(token, lit_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) tok = w.token ilabel = _Classify(gr, tok) done = p.addtoken(tok.id, tok, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DollarParen: left_token = tok lex.PushHint(Id.Op_RParen, Id.Eof_RParen) line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) c_parser = parse_ctx.MakeParserForCommandSub( line_reader, lex, Id.Eof_RParen) node = c_parser.ParseCommandSub() # A little gross: Copied from osh/word_parse.py right_token = c_parser.w_parser.cur_token cs_part = command_sub(left_token, node) cs_part.spids.append(left_token.span_id) cs_part.spids.append(right_token.span_id) typ = Id.Expr_CastedDummy opaque = cast(token, cs_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression # Now push the closing ) ilabel = _Classify(gr, right_token) done = p.addtoken(right_token.id, right_token, ilabel) assert not done # can't end the expression continue if tok.id == Id.Left_DoubleQuote: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) parts = [] # type: List[word_part_t] last_token = w_parser.ReadDoubleQuoted(left_token, parts) expr_dq_part = double_quoted(left_token, parts) typ = Id.Expr_CastedDummy opaque = cast(token, expr_dq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue if tok.id == Id.Left_DollarBrace: left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) part, last_token = w_parser.ReadBracedBracedVarSub(left_token) # It's casted word_part__BracedVarSub -> dummy -> expr__BracedVarSub! typ = Id.Expr_CastedDummy opaque = cast(token, part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue # '' and c'' if tok.id in (Id.Left_SingleQuoteRaw, Id.Left_SingleQuoteC): if tok.id == Id.Left_SingleQuoteRaw: sq_mode = lex_mode_e.SQ_Raw else: sq_mode = lex_mode_e.SQ_C left_token = tok line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok) w_parser = parse_ctx.MakeWordParser(lex, line_reader) tokens = [] # type: List[token] no_backslashes = (left_token.val == "'") last_token = w_parser.ReadSingleQuoted(sq_mode, left_token, tokens, no_backslashes) sq_part = single_quoted(left_token, tokens) typ = Id.Expr_CastedDummy opaque = cast(token, sq_part) # HACK for expr_to_ast done = p.addtoken(typ, opaque, gr.tokens[typ]) assert not done # can't end the expression continue else: # We never broke out -- EOF is too soon (how can this happen???) raise parse.ParseError("incomplete input", tok.id, tok)
def _ReadArrayLiteral(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =', token=self.cur_token) left_token = self.cur_token paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = self.parse_ctx.MakeWordParser(self.lexer, self.line_reader) words = [] # type: List[compound_word] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if w.tag_() == word_e.Token: tok = cast(Token, w) if tok.id == Id.Right_ShArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif tok.id == Id.Op_Newline: continue else: # Token p_die('Unexpected token in array literal', word=w) words.append(cast(compound_word, w)) if len(words) == 0: # a=() is empty indexed array # Needed for type safety, doh no_words = [] # type: List[word_t] node = sh_array_literal(left_token, no_words) node.spids.append(left_token.span_id) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word_.DetectAssocPair(words[0]) if pair: k, v = pair pairs = [k, v] n = len(words) for i in xrange(1, n): w2 = words[i] pair = word_.DetectAssocPair(w2) if not pair: p_die("Expected associative array pair", word=w2) k, v = pair pairs.append(k) # flat representation pairs.append(v) # invariant List? node2 = word_part.AssocArrayLiteral(left_token, pairs) node2.spids.append(paren_spid) return node2 # Brace detection for arrays but NOT associative arrays words2 = braces.BraceDetectAll(words) words3 = word_.TildeDetectAll(words2) node = sh_array_literal(left_token, words3) node.spids.append(paren_spid) return node
def _ReadArrayLiteralPart(self): # type: () -> word_part_t """ a=(1 2 3) TODO: See osh/cmd_parse.py:164 for Id.Lit_ArrayLhsOpen, for a[x++]=1 We want: A=(['x']=1 ["x"]=2 [$x$y]=3) Maybe allow this as a literal string? Because I think I've seen it before? Or maybe force people to patch to learn the rule. A=([x]=4) Starts with Lit_Other '[', and then it has Lit_ArrayLhsClose Maybe enforce that ALL have keys or NONE of have keys. """ self._Next(lex_mode_e.ShCommand) # advance past ( self._Peek() if self.cur_token.id != Id.Op_LParen: p_die('Expected ( after =, got %r', self.cur_token.val, token=self.cur_token) paren_spid = self.cur_token.span_id # MUST use a new word parser (with same lexer). w_parser = WordParser(self.parse_ctx, self.lexer, self.line_reader) words = [] while True: w = w_parser.ReadWord(lex_mode_e.ShCommand) if isinstance(w, word__TokenWord): word_id = word.CommandId(w) if word_id == Id.Right_ArrayLiteral: break # Unlike command parsing, array parsing allows embedded \n. elif word_id == Id.Op_Newline: continue else: # TokenWord p_die('Unexpected token in array literal: %r', w.token.val, word=w) assert isinstance(w, word__CompoundWord) # for MyPy words.append(w) if not words: # a=() is empty indexed array node = word_part.ArrayLiteralPart( words) # type: ignore # invariant List? node.spids.append(paren_spid) return node # If the first one is a key/value pair, then the rest are assumed to be. pair = word.DetectAssocPair(words[0]) if pair: pairs = [pair[0], pair[1]] # flat representation n = len(words) for i in xrange(1, n): w = words[i] pair = word.DetectAssocPair(w) if not pair: p_die("Expected associative array pair", word=w) pairs.append(pair[0]) # flat representation pairs.append(pair[1]) node = word_part.AssocArrayLiteral( pairs) # type: ignore # invariant List? node.spids.append(paren_spid) return node words2 = braces.BraceDetectAll(words) words3 = word.TildeDetectAll(words2) node = word_part.ArrayLiteralPart(words3) node.spids.append(paren_spid) return node