def _ReadSingleQuoted(self, lex_mode):
# type: (lex_mode_t) -> single_quoted
"""Interal method to read a word_part."""
left_token = self.cur_token
tokens = [] # type: List[Token]
# In command mode, we never disallow backslashes like '\'
self.ReadSingleQuoted(lex_mode, left_token, tokens, False)
node = single_quoted(left_token, tokens)
node.spids.append(left_token.span_id) # left '
node.spids.append(self.cur_token.span_id) # right '
return node
def _PushOilTokens(parse_ctx, gr, p, lex):
# type: (ParseContext, Grammar, parse.Parser, Lexer) -> token
"""Push tokens onto pgen2's parser.
Returns the last token so it can be reused/seen by the CommandParser.
"""
#log('keywords = %s', gr.keywords)
#log('tokens = %s', gr.tokens)
mode = lex_mode_e.Expr
mode_stack = [mode]
last_token = None
balance = 0
from core.util import log
while True:
if last_token: # e.g. left over from WordParser
tok = last_token
#log('last_token = %s', last_token)
last_token = None
else:
tok = lex.Read(mode)
#log('tok = %s', tok)
# Comments and whitespace. Newlines aren't ignored.
if meta.LookupKind(tok.id) == Kind.Ignored:
continue
# For var x = {
# a: 1, b: 2
# }
if balance > 0 and tok.id == Id.Op_Newline:
#log('*** SKIPPING NEWLINE')
continue
action = _MODE_TRANSITIONS.get((mode, tok.id))
if action == POP:
mode_stack.pop()
mode = mode_stack[-1]
balance -= 1
#log('POPPED to %s', mode)
elif action: # it's an Id
new_mode = action
mode_stack.append(new_mode)
mode = new_mode
balance += 1 # e.g. var x = $/ NEWLINE /
#log('PUSHED to %s', mode)
else:
# If we didn't already so something with the balance, look at another table.
balance += _OTHER_BALANCE.get(tok.id, 0)
#log('BALANCE after seeing %s = %d', tok.id, balance)
#if tok.id == Id.Expr_Name and tok.val in KEYWORDS:
# tok.id = KEYWORDS[tok.val]
# log('Replaced with %s', tok.id)
if tok.id.enum_id >= 256:
raise AssertionError(str(tok))
ilabel = _Classify(gr, tok)
#log('tok = %s, ilabel = %d', tok, ilabel)
if p.addtoken(tok.id.enum_id, tok, ilabel):
return tok
#
# Extra handling of the body of @() and $(). Lex in the ShCommand mode.
#
if tok.id == Id.Left_AtParen:
lex.PushHint(Id.Op_RParen, Id.Right_ShArrayLiteral)
# Blame the opening token
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.ShCommand)
if 0:
log('w = %s', w)
if isinstance(w, word__Token):
word_id = word_.CommandId(w)
if word_id == Id.Right_ShArrayLiteral:
break
elif word_id == Id.Op_Newline: # internal newlines allowed
continue
else:
# Token
p_die('Unexpected token in array literal: %r', w.token.val, word=w)
assert isinstance(w, word__Compound) # for MyPy
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word_.TildeDetectAll(words2)
typ = Id.Expr_CastedDummy.enum_id
opaque = cast(token, words3) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
# Now push the closing )
tok = w.token
ilabel = _Classify(gr, tok)
done = p.addtoken(tok.id.enum_id, tok, ilabel)
assert not done # can't end the expression
continue
if tok.id == Id.Left_DollarParen:
left_token = tok
lex.PushHint(Id.Op_RParen, Id.Eof_RParen)
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
c_parser = parse_ctx.MakeParserForCommandSub(line_reader, lex,
Id.Eof_RParen)
node = c_parser.ParseCommandSub()
# A little gross: Copied from osh/word_parse.py
right_token = c_parser.w_parser.cur_token
cs_part = command_sub(left_token, node)
cs_part.spids.append(left_token.span_id)
cs_part.spids.append(right_token.span_id)
typ = Id.Expr_CastedDummy.enum_id
opaque = cast(token, cs_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
# Now push the closing )
ilabel = _Classify(gr, right_token)
done = p.addtoken(right_token.id.enum_id, right_token, ilabel)
assert not done # can't end the expression
continue
if tok.id == Id.Left_DoubleQuote:
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
parts = [] # type: List[word_part_t]
last_token = w_parser.ReadDoubleQuoted(left_token, parts)
expr_dq_part = double_quoted(left_token, parts)
typ = Id.Expr_CastedDummy.enum_id
opaque = cast(token, expr_dq_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
if tok.id == Id.Left_DollarBrace:
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
part, last_token = w_parser.ReadBracedBracedVarSub(left_token)
# It's casted word_part__BracedVarSub -> dummy -> expr__BracedVarSub!
typ = Id.Expr_CastedDummy.enum_id
opaque = cast(token, part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
# '' and c''
if tok.id in (Id.Left_SingleQuoteRaw, Id.Left_SingleQuoteC):
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
# mode can be SQ or DollarSQ
tokens = [] # type: List[token]
no_backslashes = (left_token.val == "'")
last_token = w_parser.ReadSingleQuoted(mode, left_token, tokens,
no_backslashes)
sq_part = single_quoted(left_token, tokens)
typ = Id.Expr_CastedDummy.enum_id
opaque = cast(token, sq_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
else:
# We never broke out -- EOF is too soon (how can this happen???)
raise parse.ParseError("incomplete input", tok.id.enum_id, tok)
def _PushOilTokens(parse_ctx, gr, p, lex):
# type: (ParseContext, Grammar, parse.Parser, Lexer) -> token
"""Push tokens onto pgen2's parser.
Returns the last token so it can be reused/seen by the CommandParser.
"""
#log('keywords = %s', gr.keywords)
#log('tokens = %s', gr.tokens)
last_token = None # type: Optional[token]
balance = 0 # to ignore newlines
while True:
if last_token: # e.g. left over from WordParser
tok = last_token
#log('last_token = %s', last_token)
last_token = None
else:
tok = lex.Read(lex_mode_e.Expr)
#log('tok = %s', tok)
# Comments and whitespace. Newlines aren't ignored.
if lookup.LookupKind(tok.id) == Kind.Ignored:
continue
# For var x = {
# a: 1, b: 2
# }
if balance > 0 and tok.id == Id.Op_Newline:
#log('*** SKIPPING NEWLINE')
continue
balance += _OTHER_BALANCE.get(tok.id, 0)
#log('BALANCE after seeing %s = %d', tok.id, balance)
#if tok.id == Id.Expr_Name and tok.val in KEYWORDS:
# tok.id = KEYWORDS[tok.val]
# log('Replaced with %s', tok.id)
assert tok.id < 256, Id_str(tok.id)
ilabel = _Classify(gr, tok)
#log('tok = %s, ilabel = %d', tok, ilabel)
if p.addtoken(tok.id, tok, ilabel):
return tok
#
# Mututally recursive calls into the command/word parsers.
#
if mylib.PYTHON:
if tok.id == Id.Left_AtParen:
left_tok = tok
lex.PushHint(Id.Op_RParen, Id.Right_ShArrayLiteral)
# Blame the opening token
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
words = []
while True:
w = w_parser.ReadWord(lex_mode_e.ShCommand)
if 0:
log('w = %s', w)
if isinstance(w, word__Token):
word_id = word_.CommandId(w)
if word_id == Id.Right_ShArrayLiteral:
break
elif word_id == Id.Op_Newline: # internal newlines allowed
continue
else:
# Token
p_die('Unexpected token in array literal: %r',
w.token.val,
word=w)
assert isinstance(w, word__Compound) # for MyPy
words.append(w)
words2 = braces.BraceDetectAll(words)
words3 = word_.TildeDetectAll(words2)
typ = Id.Expr_CastedDummy
lit_part = sh_array_literal(left_tok, words3)
opaque = cast(token, lit_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
# Now push the closing )
tok = w.token
ilabel = _Classify(gr, tok)
done = p.addtoken(tok.id, tok, ilabel)
assert not done # can't end the expression
continue
if tok.id == Id.Left_DollarParen:
left_token = tok
lex.PushHint(Id.Op_RParen, Id.Eof_RParen)
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
c_parser = parse_ctx.MakeParserForCommandSub(
line_reader, lex, Id.Eof_RParen)
node = c_parser.ParseCommandSub()
# A little gross: Copied from osh/word_parse.py
right_token = c_parser.w_parser.cur_token
cs_part = command_sub(left_token, node)
cs_part.spids.append(left_token.span_id)
cs_part.spids.append(right_token.span_id)
typ = Id.Expr_CastedDummy
opaque = cast(token, cs_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
# Now push the closing )
ilabel = _Classify(gr, right_token)
done = p.addtoken(right_token.id, right_token, ilabel)
assert not done # can't end the expression
continue
if tok.id == Id.Left_DoubleQuote:
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
parts = [] # type: List[word_part_t]
last_token = w_parser.ReadDoubleQuoted(left_token, parts)
expr_dq_part = double_quoted(left_token, parts)
typ = Id.Expr_CastedDummy
opaque = cast(token, expr_dq_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
if tok.id == Id.Left_DollarBrace:
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
part, last_token = w_parser.ReadBracedBracedVarSub(left_token)
# It's casted word_part__BracedVarSub -> dummy -> expr__BracedVarSub!
typ = Id.Expr_CastedDummy
opaque = cast(token, part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
# '' and c''
if tok.id in (Id.Left_SingleQuoteRaw, Id.Left_SingleQuoteC):
if tok.id == Id.Left_SingleQuoteRaw:
sq_mode = lex_mode_e.SQ_Raw
else:
sq_mode = lex_mode_e.SQ_C
left_token = tok
line_reader = reader.DisallowedLineReader(parse_ctx.arena, tok)
w_parser = parse_ctx.MakeWordParser(lex, line_reader)
tokens = [] # type: List[token]
no_backslashes = (left_token.val == "'")
last_token = w_parser.ReadSingleQuoted(sq_mode, left_token,
tokens, no_backslashes)
sq_part = single_quoted(left_token, tokens)
typ = Id.Expr_CastedDummy
opaque = cast(token, sq_part) # HACK for expr_to_ast
done = p.addtoken(typ, opaque, gr.tokens[typ])
assert not done # can't end the expression
continue
else:
# We never broke out -- EOF is too soon (how can this happen???)
raise parse.ParseError("incomplete input", tok.id, tok)