def tokenize(self, s):
lexer = shlex.shlex(minisix.io.StringIO(s))
lexer.commenters = ''
lexer.quotes = ''
lexer.whitespace = ''
lexer.separators += self.separators
ret = []
while True:
token = lexer.get_token()
if not token:
break
elif token == '(':
ret.append(self._insideParens(lexer))
else:
ret.append(token)
return ''.join(ret)
def tokenize(self, s):
lexer = shlex.shlex(StringIO(s))
lexer.commenters = ''
lexer.quotes = ''
lexer.whitespace = ''
lexer.separators += self.separators
ret = []
while True:
token = lexer.get_token()
if not token:
break
elif token == '(':
ret.append(self._insideParens(lexer))
else:
ret.append(token)
return ''.join(ret)
