def __init__(self,
s,
strictmode=True,
expansionlimit=None,
tokenizerargs=None):
# 判断 expansionlimit为空值
# Check if the number expansionlimit is an integer:
assert expansionlimit is None or isinstance(expansionlimit, int)
# Save the context which we need to parse
self.s = s
self._strictmode = strictmode
self._expansionlimit = expansionlimit
# If there is note tokenizerargs, then tokenizerargs = None
if tokenizerargs is None:
tokenizerargs = {}
# pop() is an inbuilt function in Python
# that removes and returns last value from the list or the given index value.
self.parserstate = tokenizerargs.pop('parserstate',
state.parserstate())
self.tok = tokenizer.tokenizer(s,
parserstate=self.parserstate,
strictmode=strictmode,
**tokenizerargs)
self.redirstack = self.tok.redirstack
def __init__(self, s, strictmode=True, expansionlimit=None, tokenizerargs=None):
assert expansionlimit is None or isinstance(expansionlimit, int)
self.s = s
self._strictmode = strictmode
self._expansionlimit = expansionlimit
if tokenizerargs is None:
tokenizerargs = {}
self.parserstate = tokenizerargs.pop('parserstate', state.parserstate())
self.tok = tokenizer.tokenizer(s,
parserstate=self.parserstate,
strictmode=strictmode,
**tokenizerargs)
self.redirstack = self.tok.redirstack
def __init__(self, s, strictmode=True, expansionlimit=None, tokenizerargs=None):
assert expansionlimit is None or isinstance(expansionlimit, int)
self.s = s
self._strictmode = strictmode
self._expansionlimit = expansionlimit
if tokenizerargs is None:
tokenizerargs = {}
self.parserstate = tokenizerargs.pop('parserstate', state.parserstate())
self.tok = tokenizer.tokenizer(s,
parserstate=self.parserstate,
strictmode=strictmode,
**tokenizerargs)
self.redirstack = self.tok.redirstack
import unittest
from bashlex import tokenizer, state, flags, errors
from bashlex.tokenizer import token as t
from bashlex.tokenizer import tokentype as tt
tokenize = lambda s: list(tokenizer.tokenizer(s, state.parserstate()))
hasdollarset = set([flags.word.HASDOLLAR])
class test_tokenizer(unittest.TestCase):
def setUp(self):
if not hasattr(self, 'assertRaisesRegex'):
self.assertRaisesRegex = self.assertRaisesRegexp
def assertTokens(self, s, tokens):
result = tokenize(s)
# pop the last token if it's a new line since that gets appended
# to the input string by default and we don't really care about
# that here
if result[-1].value == '\n':
result.pop()
self.assertEqual(result, tokens)
for t in tokens:
self.assertEqual(str(t.value), s[t.lexpos:t.endlexpos])
import unittest
from bashlex import tokenizer, state, flags, errors
from bashlex.tokenizer import token as t
from bashlex.tokenizer import tokentype as tt
tokenize = lambda s: list(tokenizer.tokenizer(s, state.parserstate()))
hasdollarset = set([flags.word.HASDOLLAR])
class test_tokenizer(unittest.TestCase):
def assertTokens(self, s, tokens):
result = tokenize(s)
# pop the last token if it's a new line since that gets appended
# to the input string by default and we don't really care about
# that here
if result[-1].value == '\n':
result.pop()
self.assertEquals(result, tokens)
for t in tokens:
self.assertEquals(str(t.value), s[t.lexpos:t.endlexpos])
def test_empty_string(self):
self.assertEquals(len(tokenize('')), 0)
def test_simple(self):
s = 'a b'