def _readtokenword(self, c):
d = {}
d['all_digit_token'] = c.isdigit()
d['dollar_present'] = d['quoted'] = d['pass_next_character'] = d['compound_assignment'] = False
tokenword = []
def handleshellquote():
self._push_delimiter(c)
try:
ttok = self._parse_matched_pair(c, c, c, parsingcommand=(c == '`'))
finally:
self._pop_delimiter()
tokenword.append(c)
tokenword.extend(ttok)
d['all_digit_token'] = False
d['quoted'] = True
if not d['dollar_present']:
d['dollar_present'] = c == '"' and '$' in ttok
def handleshellexp():
peek_char = self._getc()
if peek_char == '(' or (c == '$' and peek_char in '{['):
# try:
if peek_char == '{':
ttok = self._parse_matched_pair(cd, '{', '}', firstclose=True, dolbrace=True)
elif peek_char == '(':
self._push_delimiter(peek_char)
ttok = self._parse_comsub(cd, '(', ')', parsingcommand=True)
self._pop_delimiter()
else:
ttok = self._parse_matched_pair(cd, '[', ']')
# except MatchedPairError:
# return -1
tokenword.append(c)
tokenword.append(peek_char)
tokenword.extend(ttok)
d['dollar_present'] = True
d['all_digit_token'] = False
# goto next_character
elif c == '$' and peek_char in '\'"':
self._push_delimiter(peek_char)
try:
ttok = self._parse_matched_pair(peek_char, peek_char, peek_char,
allowesc=(peek_char == "'"))
# except MatchedPairError:
# return -1
finally:
self._pop_delimiter()
#if peek_char == "'":
# # XXX ansiexpand
# ttok = shutils.single_quote(ttok)
#else:
# ttok = shutils.double_quote(ttok)
tokenword.append(c)
tokenword.append(peek_char)
tokenword.extend(ttok)
d['quoted'] = True
d['all_digit_token'] = False
# goto next_character
elif c == '$' and peek_char == '$':
tokenword.append('$')
tokenword.append('$')
d['dollar_present'] = True
d['all_digit_token'] = False
# goto next_character
else:
self._ungetc(peek_char)
return True
# 4699 ARRAY_VARS
def handleescapedchar():
tokenword.append(c)
d['all_digit_token'] &= c.isdigit()
if not d['dollar_present']:
d['dollar_present'] = c == '$'
while True:
if c is None:
break
if d['pass_next_character']:
d['pass_next_character'] = False
handleescapedchar()
# goto escaped_character
else:
cd = self._current_delimiter()
gotonext = False
if c == '\\':
peek_char = self._getc(False)
if peek_char == '\n':
c = '\n'
gotonext = True
# goto next_character
else:
self._ungetc(peek_char)
if (cd is None or cd == '`' or
(cd == '"' and peek_char is not None and
'dquote' in sh_syntaxtab[peek_char])):
d['pass_next_character'] = True
d['quoted'] = True
handleescapedchar()
gotonext = True
# goto got_character
elif _shellquote(c):
handleshellquote()
gotonext = True
# goto next_character
# XXX 4542
# XXX 4567
elif _shellexp(c):
gotonext = not handleshellexp()
# XXX 4699
if not gotonext:
if _shellbreak(c):
self._ungetc(c)
break
else:
handleescapedchar()
# got_character
# got_escaped_character
# tokenword.append(c)
# all_digit_token &= c.isdigit()
# if not dollar_present:
# dollar_present = c == '$'
# next_character
cd = self._current_delimiter()
c = self._getc(cd != "'" and not d['pass_next_character'])
# got_token
self._recordpos()
tokenword = ''.join(tokenword)
if d['all_digit_token'] and (c in '<>' or self._last_read_token.ttype in (tokentype.LESS_AND, tokentype.GREATER_AND)) and shutils.legal_number(tokenword):
return self._createtoken(tokentype.NUMBER, int(tokenword))
# 4811
specialtokentype = self._specialcasetokens(tokenword)
if specialtokentype:
return self._createtoken(specialtokentype, tokenword)
if not d['dollar_present'] and not d['quoted'] and self._reserved_word_acceptable(self._last_read_token):
if tokenword in valid_reserved_first_command:
ttype = valid_reserved_first_command[tokenword]
ps = self._parserstate
if ps & parserflags.CASEPAT and ttype != tokentype.ESAC:
pass
elif ttype == tokentype.TIME and not self._time_command_acceptable():
pass
elif ttype == tokentype.ESAC:
ps.discard(parserflags.CASEPAT)
ps.discard(parserflags.CASESTMT)
elif ttype == tokentype.CASE:
ps.add(parserflags.CASESTMT)
elif ttype == tokentype.COND_END:
ps.discard(parserflags.CONDCMD)
ps.discard(parserflags.CONDEXPR)
elif ttype == tokentype.COND_START:
ps.add(parserflags.CONDCMD)
elif ttype == tokentype.LEFT_CURLY:
self._open_brace_count += 1
elif ttype == tokentype.RIGHT_CURLY and self._open_brace_count:
self._open_brace_count -= 1
return self._createtoken(ttype, tokenword)
tokenword = self._createtoken(tokentype.WORD, tokenword, utils.typedset(wordflags))
if d['dollar_present']:
tokenword.flags.add(wordflags.HASDOLLAR)
if d['quoted']:
tokenword.flags.add(wordflags.QUOTED)
if d['compound_assignment'] and tokenword[-1] == ')':
tokenword.flags.add(wordflags.COMPASSIGN)
if self._is_assignment(tokenword.value, bool(self._parserstate & parserflags.COMPASSIGN)):
tokenword.flags.add(wordflags.ASSIGNMENT)
if self._assignment_acceptable(self._last_read_token):
tokenword.flags.add(wordflags.NOSPLIT)
if self._parserstate & parserflags.COMPASSIGN:
tokenword.flags.add(wordflags.NOGLOB)
# XXX 4865
if self._command_token_position(self._last_read_token):
pass
if tokenword.value[0] == '{' and tokenword.value[-1] == '}' and c in '<>':
if shutils.legal_identifier(tokenword.value[1:]):
# XXX is this needed?
tokenword.value = tokenword.value[1:]
tokenword.ttype = tokentype.REDIR_WORD
return tokenword
if len(tokenword.flags & set([wordflags.ASSIGNMENT, wordflags.NOSPLIT])) == 2:
tokenword.ttype = tokentype.ASSIGNMENT_WORD
if self._last_read_token.ttype == tokentype.FUNCTION:
self._parserstate.add(parserflags.ALLOWOPNBRC)
self._function_dstart = self._line_number
elif self._last_read_token.ttype in (tokentype.CASE, tokentype.SELECT, tokentype.FOR):
pass # XXX 4907
return tokenword
def _readtokenword(self, c):
d = {}
d['all_digit_token'] = c.isdigit()
d['dollar_present'] = d['quoted'] = d['pass_next_character'] = d[
'compound_assignment'] = False
tokenword = []
def handleshellquote():
self._push_delimiter(c)
try:
ttok = self._parse_matched_pair(c,
c,
c,
parsingcommand=(c == '`'))
finally:
self._pop_delimiter()
tokenword.append(c)
tokenword.extend(ttok)
d['all_digit_token'] = False
d['quoted'] = True
if not d['dollar_present']:
d['dollar_present'] = c == '"' and '$' in ttok
def handleshellexp():
peek_char = self._getc()
if peek_char == '(' or (c == '$' and peek_char in '{['):
# try:
if peek_char == '{':
ttok = self._parse_matched_pair(cd,
'{',
'}',
firstclose=True,
dolbrace=True)
elif peek_char == '(':
self._push_delimiter(peek_char)
ttok = self._parse_comsub(cd,
'(',
')',
parsingcommand=True)
self._pop_delimiter()
else:
ttok = self._parse_matched_pair(cd, '[', ']')
# except MatchedPairError:
# return -1
tokenword.append(c)
tokenword.append(peek_char)
tokenword.extend(ttok)
d['dollar_present'] = True
d['all_digit_token'] = False
# goto next_character
elif c == '$' and peek_char in '\'"':
self._push_delimiter(peek_char)
try:
ttok = self._parse_matched_pair(
peek_char,
peek_char,
peek_char,
allowesc=(peek_char == "'"))
# except MatchedPairError:
# return -1
finally:
self._pop_delimiter()
#if peek_char == "'":
# # XXX ansiexpand
# ttok = shutils.single_quote(ttok)
#else:
# ttok = shutils.double_quote(ttok)
tokenword.append(c)
tokenword.append(peek_char)
tokenword.extend(ttok)
d['quoted'] = True
d['all_digit_token'] = False
# goto next_character
elif c == '$' and peek_char == '$':
tokenword.append('$')
tokenword.append('$')
d['dollar_present'] = True
d['all_digit_token'] = False
# goto next_character
else:
self._ungetc(peek_char)
return True
# bashlex/parse.y L4699 ARRAY_VARS
def handleescapedchar():
tokenword.append(c)
d['all_digit_token'] &= c.isdigit()
if not d['dollar_present']:
d['dollar_present'] = c == '$'
while True:
if c is None:
break
if d['pass_next_character']:
d['pass_next_character'] = False
handleescapedchar()
# goto escaped_character
else:
cd = self._current_delimiter()
gotonext = False
if c == '\\':
peek_char = self._getc(False)
if peek_char == '\n':
c = '\n'
gotonext = True
# goto next_character
else:
self._ungetc(peek_char)
if (cd is None or cd == '`'
or (cd == '"' and peek_char is not None
and 'dquote' in sh_syntaxtab[peek_char])):
d['pass_next_character'] = True
d['quoted'] = True
handleescapedchar()
gotonext = True
# goto got_character
elif _shellquote(c):
handleshellquote()
gotonext = True
# goto next_character
# bashlex/parse.y L4542
# bashlex/parse.y L4567
elif _shellexp(c):
gotonext = not handleshellexp()
# bashlex/parse.y L4699
if not gotonext:
if _shellbreak(c):
self._ungetc(c)
break
else:
handleescapedchar()
# got_character
# got_escaped_character
# tokenword.append(c)
# all_digit_token &= c.isdigit()
# if not dollar_present:
# dollar_present = c == '$'
# next_character
cd = self._current_delimiter()
c = self._getc(cd != "'" and not d['pass_next_character'])
# got_token
self._recordpos()
tokenword = ''.join(tokenword)
if d['all_digit_token'] and (c in '<>' or self._last_read_token.ttype
in (tokentype.LESS_AND,
tokentype.GREATER_AND)
) and shutils.legal_number(tokenword):
return self._createtoken(tokentype.NUMBER, int(tokenword))
# bashlex/parse.y L4811
specialtokentype = self._specialcasetokens(tokenword)
if specialtokentype:
return self._createtoken(specialtokentype, tokenword)
if not d['dollar_present'] and not d[
'quoted'] and self._reserved_word_acceptable(
self._last_read_token):
if tokenword in valid_reserved_first_command:
ttype = valid_reserved_first_command[tokenword]
ps = self._parserstate
if ps & parserflags.CASEPAT and ttype != tokentype.ESAC:
pass
elif ttype == tokentype.TIME and not self._time_command_acceptable(
):
pass
elif ttype == tokentype.ESAC:
ps.discard(parserflags.CASEPAT)
ps.discard(parserflags.CASESTMT)
elif ttype == tokentype.CASE:
ps.add(parserflags.CASESTMT)
elif ttype == tokentype.COND_END:
ps.discard(parserflags.CONDCMD)
ps.discard(parserflags.CONDEXPR)
elif ttype == tokentype.COND_START:
ps.add(parserflags.CONDCMD)
elif ttype == tokentype.LEFT_CURLY:
self._open_brace_count += 1
elif ttype == tokentype.RIGHT_CURLY and self._open_brace_count:
self._open_brace_count -= 1
return self._createtoken(ttype, tokenword)
tokenword = self._createtoken(tokentype.WORD, tokenword,
utils.typedset(wordflags))
if d['dollar_present']:
tokenword.flags.add(wordflags.HASDOLLAR)
if d['quoted']:
tokenword.flags.add(wordflags.QUOTED)
if d['compound_assignment'] and tokenword[-1] == ')':
tokenword.flags.add(wordflags.COMPASSIGN)
if self._is_assignment(
tokenword.value,
bool(self._parserstate & parserflags.COMPASSIGN)):
tokenword.flags.add(wordflags.ASSIGNMENT)
if self._assignment_acceptable(self._last_read_token):
tokenword.flags.add(wordflags.NOSPLIT)
if self._parserstate & parserflags.COMPASSIGN:
tokenword.flags.add(wordflags.NOGLOB)
# bashlex/parse.y L4865
if self._command_token_position(self._last_read_token):
pass
if tokenword.value[0] == '{' and tokenword.value[
-1] == '}' and c in '<>':
if shutils.legal_identifier(tokenword.value[1:]):
# XXX is this needed?
tokenword.value = tokenword.value[1:]
tokenword.ttype = tokentype.REDIR_WORD
return tokenword
if len(tokenword.flags
& set([wordflags.ASSIGNMENT, wordflags.NOSPLIT])) == 2:
tokenword.ttype = tokentype.ASSIGNMENT_WORD
if self._last_read_token.ttype == tokentype.FUNCTION:
self._parserstate.add(parserflags.ALLOWOPNBRC)
self._function_dstart = self._line_number
elif self._last_read_token.ttype in (tokentype.CASE, tokentype.SELECT,
tokentype.FOR):
pass # bashlex/parse.y L4907
return tokenword
from bashlex import flags, utils parserstate = lambda: utils.typedset(flags.parser)
