def _ReadArithWord(self):
"""Helper function for ReadArithWord."""
#assert self.token_type != Id.Undefined_Tok
self._Peek()
#print('_ReadArithWord', self.cur_token)
if self.token_kind == Kind.Unknown:
self.AddErrorContext("Unknown token in arith context: %s",
self.cur_token,
token=self.cur_token)
return None, False
elif self.token_kind == Kind.Eof:
# Just return EOF token
w = ast.TokenWord(self.cur_token)
return w, False
#self.AddErrorContext("Unexpected EOF in arith context: %s",
# self.cur_token, token=self.cur_token)
#return None, False
elif self.token_kind == Kind.Ignored:
# Space should be ignored. TODO: change this to SPACE_SPACE and
# SPACE_NEWLINE? or SPACE_TOK.
self._Next(lex_mode_e.ARITH)
return None, True # Tell wrapper to try again
elif self.token_kind in (Kind.Arith, Kind.Right):
# Id.Right_ArithSub IS just a normal token, handled by ArithParser
self._Next(lex_mode_e.ARITH)
w = ast.TokenWord(self.cur_token)
return w, False
elif self.token_kind in (Kind.Lit, Kind.Left):
w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH)
if not w:
return None, True
return w, False
elif self.token_kind == Kind.VSub:
part = ast.SimpleVarSub(self.cur_token)
self._Next(lex_mode_e.ARITH)
w = ast.CompoundWord([part])
return w, False
else:
self._BadToken("Unexpected token parsing arith sub: %s",
self.cur_token)
return None, False
raise AssertionError("Shouldn't get here")
def _ReadLikeDQ(self, left_dq_token, out_parts):
"""
Args:
left_dq_token: A token if we are reading a double quoted part, or None if
we're reading a here doc.
out_parts: list of word_part to append to
"""
done = False
while not done:
self._Next(lex_mode_e.DQ)
self._Peek()
if self.token_kind == Kind.Lit:
if self.token_type == Id.Lit_EscapedChar:
part = ast.EscapedLiteralPart(self.cur_token)
else:
part = ast.LiteralPart(self.cur_token)
out_parts.append(part)
elif self.token_kind == Kind.Left:
part = self._ReadDoubleQuotedLeftParts()
out_parts.append(part)
elif self.token_kind == Kind.VSub:
part = ast.SimpleVarSub(self.cur_token)
out_parts.append(part)
elif self.token_kind == Kind.Right:
assert self.token_type == Id.Right_DoubleQuote, self.token_type
if left_dq_token:
done = True
else:
# In a here doc, the right quote is literal!
out_parts.append(ast.LiteralPart(self.cur_token))
elif self.token_kind == Kind.Eof:
if left_dq_token:
p_die(
'Unexpected EOF reading double-quoted string that began here',
token=left_dq_token)
else: # here docs will have an EOF in their token stream
done = True
else:
raise AssertionError(self.cur_token)
def _ReadArithWord(self):
"""Helper function for ReadArithWord."""
self._Peek()
if self.token_kind == Kind.Unknown:
p_die('Unexpected token in arithmetic context',
token=self.cur_token)
elif self.token_kind == Kind.Eof:
# Just return EOF token
w = ast.TokenWord(self.cur_token)
return w, False
elif self.token_kind == Kind.Ignored:
# Space should be ignored. TODO: change this to SPACE_SPACE and
# SPACE_NEWLINE? or SPACE_TOK.
self._Next(lex_mode_e.ARITH)
return None, True # Tell wrapper to try again
elif self.token_kind in (Kind.Arith, Kind.Right):
# Id.Right_ArithSub IS just a normal token, handled by ArithParser
self._Next(lex_mode_e.ARITH)
w = ast.TokenWord(self.cur_token)
return w, False
elif self.token_kind in (Kind.Lit, Kind.Left):
w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH)
return w, False
elif self.token_kind == Kind.VSub:
part = ast.SimpleVarSub(self.cur_token)
self._Next(lex_mode_e.ARITH)
w = ast.CompoundWord([part])
return w, False
else:
assert False, ("Unexpected token parsing arith sub: %s" %
self.cur_token)
raise AssertionError("Shouldn't get here")
def _ReadCompoundWord(self,
eof_type=Id.Undefined_Tok,
lex_mode=lex_mode_e.OUTER,
empty_ok=True):
"""
Precondition: Looking at the first token of the first word part
Postcondition: Looking at the token after, e.g. space or operator
NOTE: eof_type is necessary because / is a literal, i.e. Lit_Slash, but it
could be an operator delimiting a compound word. Can we change lexer modes
and remove this special case?
"""
#print('_ReadCompoundWord', lex_mode)
word = ast.CompoundWord()
num_parts = 0
done = False
while not done:
allow_done = empty_ok or num_parts != 0
self._Peek()
#print('CW',self.cur_token)
if allow_done and self.token_type == eof_type:
done = True # e.g. for ${foo//pat/replace}
# Keywords like "for" are treated like literals
elif self.token_kind in (Kind.Lit, Kind.KW, Kind.Assign,
Kind.ControlFlow, Kind.BoolUnary,
Kind.BoolBinary):
if self.token_type == Id.Lit_EscapedChar:
part = ast.EscapedLiteralPart(self.cur_token)
else:
part = ast.LiteralPart(self.cur_token)
#part.xspans.append(self.cur_token.span_id)
word.parts.append(part)
if self.token_type == Id.Lit_VarLike:
#print('@', self.cursor)
#print('@', self.cur_token)
t = self.lexer.LookAhead(lex_mode_e.OUTER)
if t.id == Id.Op_LParen:
self.lexer.PushHint(Id.Op_RParen,
Id.Right_ArrayLiteral)
part2 = self._ReadArrayLiteralPart()
if not part2:
self.AddErrorContext(
'_ReadArrayLiteralPart failed')
return False
word.parts.append(part2)
elif self.token_kind == Kind.VSub:
part = ast.SimpleVarSub(self.cur_token)
word.parts.append(part)
elif self.token_kind == Kind.ExtGlob:
part = self._ReadExtGlobPart()
if not part:
return None
word.parts.append(part)
elif self.token_kind == Kind.Left:
#print('_ReadLeftParts')
part = self._ReadLeftParts()
if not part:
return None
word.parts.append(part)
# NOT done yet, will advance below
elif self.token_kind == Kind.Right:
# Still part of the word; will be done on the next iter.
if self.token_type == Id.Right_DoubleQuote:
pass
elif self.token_type == Id.Right_CommandSub:
pass
elif self.token_type == Id.Right_Subshell:
# LEXER HACK for (case x in x) ;; esac )
assert self.next_lex_mode is None # Rewind before it's used
if self.lexer.MaybeUnreadOne():
self.lexer.PushHint(Id.Op_RParen, Id.Right_Subshell)
self._Next(lex_mode)
done = True
else:
done = True
elif self.token_kind == Kind.Ignored:
done = True
else:
# LEXER HACK for unbalanced case clause. 'case foo in esac' is valid,
# so to test for ESAC, we can read ) before getting a chance to
# PushHint(Id.Op_RParen, Id.Right_CasePat). So here we unread one
# token and do it again.
# We get Id.Op_RParen at top level: case x in x) ;; esac
# We get Id.Eof_RParen inside ComSub: $(case x in x) ;; esac )
if self.token_type in (Id.Op_RParen, Id.Eof_RParen):
assert self.next_lex_mode is None # Rewind before it's used
if self.lexer.MaybeUnreadOne():
if self.token_type == Id.Eof_RParen:
# Redo translation
self.lexer.PushHint(Id.Op_RParen, Id.Eof_RParen)
self._Next(lex_mode)
done = True # anything we don't recognize means we're done
if not done:
self._Next(lex_mode)
num_parts += 1
return word
def _ReadDoubleQuotedPart(self, eof_type=Id.Undefined_Tok, here_doc=False):
"""
Args:
eof_type: for stopping at }, Id.Lit_RBrace
here_doc: Whether we are reading in a here doc context
Also ${foo%%a b c} # treat this as double quoted. until you hit
"""
quoted_part = ast.DoubleQuotedPart()
left_spid = const.NO_INTEGER
right_spid = const.NO_INTEGER # gets set later
if self.cur_token is not None: # None in here doc case
left_spid = self.cur_token.span_id
done = False
while not done:
self._Next(lex_mode_e.DQ)
self._Peek()
#print(self.cur_token)
if self.token_type == eof_type: # e.g. stop at }
done = True
continue
elif self.token_kind == Kind.Lit:
if self.token_type == Id.Lit_EscapedChar:
part = ast.EscapedLiteralPart(self.cur_token)
else:
part = ast.LiteralPart(self.cur_token)
quoted_part.parts.append(part)
elif self.token_kind == Kind.Left:
part = self._ReadDoubleQuotedLeftParts()
if not part:
return None
quoted_part.parts.append(part)
elif self.token_kind == Kind.VSub:
part = ast.SimpleVarSub(self.cur_token)
quoted_part.parts.append(part)
elif self.token_kind == Kind.Right:
assert self.token_type == Id.Right_DoubleQuote
if here_doc:
# Turn Id.Right_DoubleQuote into a literal part
quoted_part.parts.append(ast.LiteralPart(self.cur_token))
else:
done = True # assume Id.Right_DoubleQuote
right_spid = self.cur_token.span_id
elif self.token_kind == Kind.Eof:
if here_doc: # here docs will have an EOF in their token stream
done = True
else:
self.AddErrorContext(
'Unexpected EOF reading double-quoted string that began here',
span_id=left_spid)
return False
else:
raise AssertionError(self.cur_token)
quoted_part.spids.extend((left_spid, right_spid))
return quoted_part