def _readtokenword(self, c): d = {} d['all_digit_token'] = c.isdigit() d['dollar_present'] = d['quoted'] = d['pass_next_character'] = d['compound_assignment'] = False tokenword = [] def handleshellquote(): self._push_delimiter(c) try: ttok = self._parse_matched_pair(c, c, c, parsingcommand=(c == '`')) finally: self._pop_delimiter() tokenword.append(c) tokenword.extend(ttok) d['all_digit_token'] = False d['quoted'] = True if not d['dollar_present']: d['dollar_present'] = c == '"' and '$' in ttok def handleshellexp(): peek_char = self._getc() if peek_char == '(' or (c == '$' and peek_char in '{['): # try: if peek_char == '{': ttok = self._parse_matched_pair(cd, '{', '}', firstclose=True, dolbrace=True) elif peek_char == '(': self._push_delimiter(peek_char) ttok = self._parse_comsub(cd, '(', ')', parsingcommand=True) self._pop_delimiter() else: ttok = self._parse_matched_pair(cd, '[', ']') # except MatchedPairError: # return -1 tokenword.append(c) tokenword.append(peek_char) tokenword.extend(ttok) d['dollar_present'] = True d['all_digit_token'] = False # goto next_character elif c == '$' and peek_char in '\'"': self._push_delimiter(peek_char) try: ttok = self._parse_matched_pair(peek_char, peek_char, peek_char, allowesc=(peek_char == "'")) # except MatchedPairError: # return -1 finally: self._pop_delimiter() #if peek_char == "'": # # XXX ansiexpand # ttok = shutils.single_quote(ttok) #else: # ttok = shutils.double_quote(ttok) tokenword.append(c) tokenword.append(peek_char) tokenword.extend(ttok) d['quoted'] = True d['all_digit_token'] = False # goto next_character elif c == '$' and peek_char == '$': tokenword.append('$') tokenword.append('$') d['dollar_present'] = True d['all_digit_token'] = False # goto next_character else: self._ungetc(peek_char) return True # 4699 ARRAY_VARS def handleescapedchar(): tokenword.append(c) d['all_digit_token'] &= c.isdigit() if not d['dollar_present']: d['dollar_present'] = c == '$' while True: if c is None: break if d['pass_next_character']: d['pass_next_character'] = False handleescapedchar() # goto escaped_character else: cd = self._current_delimiter() gotonext = False if c == '\\': peek_char = self._getc(False) if peek_char == '\n': c = '\n' gotonext = True # goto next_character else: self._ungetc(peek_char) if (cd is None or cd == '`' or (cd == '"' and peek_char is not None and 'dquote' in sh_syntaxtab[peek_char])): d['pass_next_character'] = True d['quoted'] = True handleescapedchar() gotonext = True # goto got_character elif _shellquote(c): handleshellquote() gotonext = True # goto next_character # XXX 4542 # XXX 4567 elif _shellexp(c): gotonext = not handleshellexp() # XXX 4699 if not gotonext: if _shellbreak(c): self._ungetc(c) break else: handleescapedchar() # got_character # got_escaped_character # tokenword.append(c) # all_digit_token &= c.isdigit() # if not dollar_present: # dollar_present = c == '$' # next_character cd = self._current_delimiter() c = self._getc(cd != "'" and not d['pass_next_character']) # got_token self._recordpos() tokenword = ''.join(tokenword) if d['all_digit_token'] and (c in '<>' or self._last_read_token.ttype in (tokentype.LESS_AND, tokentype.GREATER_AND)) and shutils.legal_number(tokenword): return self._createtoken(tokentype.NUMBER, int(tokenword)) # 4811 specialtokentype = self._specialcasetokens(tokenword) if specialtokentype: return self._createtoken(specialtokentype, tokenword) if not d['dollar_present'] and not d['quoted'] and self._reserved_word_acceptable(self._last_read_token): if tokenword in valid_reserved_first_command: ttype = valid_reserved_first_command[tokenword] ps = self._parserstate if ps & parserflags.CASEPAT and ttype != tokentype.ESAC: pass elif ttype == tokentype.TIME and not self._time_command_acceptable(): pass elif ttype == tokentype.ESAC: ps.discard(parserflags.CASEPAT) ps.discard(parserflags.CASESTMT) elif ttype == tokentype.CASE: ps.add(parserflags.CASESTMT) elif ttype == tokentype.COND_END: ps.discard(parserflags.CONDCMD) ps.discard(parserflags.CONDEXPR) elif ttype == tokentype.COND_START: ps.add(parserflags.CONDCMD) elif ttype == tokentype.LEFT_CURLY: self._open_brace_count += 1 elif ttype == tokentype.RIGHT_CURLY and self._open_brace_count: self._open_brace_count -= 1 return self._createtoken(ttype, tokenword) tokenword = self._createtoken(tokentype.WORD, tokenword, utils.typedset(wordflags)) if d['dollar_present']: tokenword.flags.add(wordflags.HASDOLLAR) if d['quoted']: tokenword.flags.add(wordflags.QUOTED) if d['compound_assignment'] and tokenword[-1] == ')': tokenword.flags.add(wordflags.COMPASSIGN) if self._is_assignment(tokenword.value, bool(self._parserstate & parserflags.COMPASSIGN)): tokenword.flags.add(wordflags.ASSIGNMENT) if self._assignment_acceptable(self._last_read_token): tokenword.flags.add(wordflags.NOSPLIT) if self._parserstate & parserflags.COMPASSIGN: tokenword.flags.add(wordflags.NOGLOB) # XXX 4865 if self._command_token_position(self._last_read_token): pass if tokenword.value[0] == '{' and tokenword.value[-1] == '}' and c in '<>': if shutils.legal_identifier(tokenword.value[1:]): # XXX is this needed? tokenword.value = tokenword.value[1:] tokenword.ttype = tokentype.REDIR_WORD return tokenword if len(tokenword.flags & set([wordflags.ASSIGNMENT, wordflags.NOSPLIT])) == 2: tokenword.ttype = tokentype.ASSIGNMENT_WORD if self._last_read_token.ttype == tokentype.FUNCTION: self._parserstate.add(parserflags.ALLOWOPNBRC) self._function_dstart = self._line_number elif self._last_read_token.ttype in (tokentype.CASE, tokentype.SELECT, tokentype.FOR): pass # XXX 4907 return tokenword
def _readtokenword(self, c): d = {} d['all_digit_token'] = c.isdigit() d['dollar_present'] = d['quoted'] = d['pass_next_character'] = d[ 'compound_assignment'] = False tokenword = [] def handleshellquote(): self._push_delimiter(c) try: ttok = self._parse_matched_pair(c, c, c, parsingcommand=(c == '`')) finally: self._pop_delimiter() tokenword.append(c) tokenword.extend(ttok) d['all_digit_token'] = False d['quoted'] = True if not d['dollar_present']: d['dollar_present'] = c == '"' and '$' in ttok def handleshellexp(): peek_char = self._getc() if peek_char == '(' or (c == '$' and peek_char in '{['): # try: if peek_char == '{': ttok = self._parse_matched_pair(cd, '{', '}', firstclose=True, dolbrace=True) elif peek_char == '(': self._push_delimiter(peek_char) ttok = self._parse_comsub(cd, '(', ')', parsingcommand=True) self._pop_delimiter() else: ttok = self._parse_matched_pair(cd, '[', ']') # except MatchedPairError: # return -1 tokenword.append(c) tokenword.append(peek_char) tokenword.extend(ttok) d['dollar_present'] = True d['all_digit_token'] = False # goto next_character elif c == '$' and peek_char in '\'"': self._push_delimiter(peek_char) try: ttok = self._parse_matched_pair( peek_char, peek_char, peek_char, allowesc=(peek_char == "'")) # except MatchedPairError: # return -1 finally: self._pop_delimiter() #if peek_char == "'": # # XXX ansiexpand # ttok = shutils.single_quote(ttok) #else: # ttok = shutils.double_quote(ttok) tokenword.append(c) tokenword.append(peek_char) tokenword.extend(ttok) d['quoted'] = True d['all_digit_token'] = False # goto next_character elif c == '$' and peek_char == '$': tokenword.append('$') tokenword.append('$') d['dollar_present'] = True d['all_digit_token'] = False # goto next_character else: self._ungetc(peek_char) return True # bashlex/parse.y L4699 ARRAY_VARS def handleescapedchar(): tokenword.append(c) d['all_digit_token'] &= c.isdigit() if not d['dollar_present']: d['dollar_present'] = c == '$' while True: if c is None: break if d['pass_next_character']: d['pass_next_character'] = False handleescapedchar() # goto escaped_character else: cd = self._current_delimiter() gotonext = False if c == '\\': peek_char = self._getc(False) if peek_char == '\n': c = '\n' gotonext = True # goto next_character else: self._ungetc(peek_char) if (cd is None or cd == '`' or (cd == '"' and peek_char is not None and 'dquote' in sh_syntaxtab[peek_char])): d['pass_next_character'] = True d['quoted'] = True handleescapedchar() gotonext = True # goto got_character elif _shellquote(c): handleshellquote() gotonext = True # goto next_character # bashlex/parse.y L4542 # bashlex/parse.y L4567 elif _shellexp(c): gotonext = not handleshellexp() # bashlex/parse.y L4699 if not gotonext: if _shellbreak(c): self._ungetc(c) break else: handleescapedchar() # got_character # got_escaped_character # tokenword.append(c) # all_digit_token &= c.isdigit() # if not dollar_present: # dollar_present = c == '$' # next_character cd = self._current_delimiter() c = self._getc(cd != "'" and not d['pass_next_character']) # got_token self._recordpos() tokenword = ''.join(tokenword) if d['all_digit_token'] and (c in '<>' or self._last_read_token.ttype in (tokentype.LESS_AND, tokentype.GREATER_AND) ) and shutils.legal_number(tokenword): return self._createtoken(tokentype.NUMBER, int(tokenword)) # bashlex/parse.y L4811 specialtokentype = self._specialcasetokens(tokenword) if specialtokentype: return self._createtoken(specialtokentype, tokenword) if not d['dollar_present'] and not d[ 'quoted'] and self._reserved_word_acceptable( self._last_read_token): if tokenword in valid_reserved_first_command: ttype = valid_reserved_first_command[tokenword] ps = self._parserstate if ps & parserflags.CASEPAT and ttype != tokentype.ESAC: pass elif ttype == tokentype.TIME and not self._time_command_acceptable( ): pass elif ttype == tokentype.ESAC: ps.discard(parserflags.CASEPAT) ps.discard(parserflags.CASESTMT) elif ttype == tokentype.CASE: ps.add(parserflags.CASESTMT) elif ttype == tokentype.COND_END: ps.discard(parserflags.CONDCMD) ps.discard(parserflags.CONDEXPR) elif ttype == tokentype.COND_START: ps.add(parserflags.CONDCMD) elif ttype == tokentype.LEFT_CURLY: self._open_brace_count += 1 elif ttype == tokentype.RIGHT_CURLY and self._open_brace_count: self._open_brace_count -= 1 return self._createtoken(ttype, tokenword) tokenword = self._createtoken(tokentype.WORD, tokenword, utils.typedset(wordflags)) if d['dollar_present']: tokenword.flags.add(wordflags.HASDOLLAR) if d['quoted']: tokenword.flags.add(wordflags.QUOTED) if d['compound_assignment'] and tokenword[-1] == ')': tokenword.flags.add(wordflags.COMPASSIGN) if self._is_assignment( tokenword.value, bool(self._parserstate & parserflags.COMPASSIGN)): tokenword.flags.add(wordflags.ASSIGNMENT) if self._assignment_acceptable(self._last_read_token): tokenword.flags.add(wordflags.NOSPLIT) if self._parserstate & parserflags.COMPASSIGN: tokenword.flags.add(wordflags.NOGLOB) # bashlex/parse.y L4865 if self._command_token_position(self._last_read_token): pass if tokenword.value[0] == '{' and tokenword.value[ -1] == '}' and c in '<>': if shutils.legal_identifier(tokenword.value[1:]): # XXX is this needed? tokenword.value = tokenword.value[1:] tokenword.ttype = tokentype.REDIR_WORD return tokenword if len(tokenword.flags & set([wordflags.ASSIGNMENT, wordflags.NOSPLIT])) == 2: tokenword.ttype = tokentype.ASSIGNMENT_WORD if self._last_read_token.ttype == tokentype.FUNCTION: self._parserstate.add(parserflags.ALLOWOPNBRC) self._function_dstart = self._line_number elif self._last_read_token.ttype in (tokentype.CASE, tokentype.SELECT, tokentype.FOR): pass # bashlex/parse.y L4907 return tokenword
from bashlex import flags, utils parserstate = lambda: utils.typedset(flags.parser)