class TestLexer(TestCase):
def setUp(self):
self.lexer = BentoLexer()
def _test(self, data, ref):
self.lexer.input(data)
self._test_impl(data, ref)
def _test_impl(self, data, ref):
res = []
while True:
tok = self.lexer.token()
if not tok:
break
res.append(tok.type)
if is_string(ref):
ref = split(ref)
try:
self.assertEqual(res, ref)
except AssertionError:
e = extract_exception()
cnt = 0
for i, j in zip(res, ref):
if not i == j:
break
cnt += 1
print(("Break at index %d" % cnt))
raise e
def _get_tokens(self, data):
self.lexer.input(data)
return list(self.lexer)
class TestLexer(TestCase):
def setUp(self):
self.lexer = BentoLexer()
def _test(self, data, ref):
self.lexer.input(data)
self._test_impl(data, ref)
def _test_impl(self, data, ref):
res = []
while True:
tok = self.lexer.token()
if not tok:
break
res.append(tok.type)
if is_string(ref):
ref = split(ref)
try:
self.assertEqual(res, ref)
except AssertionError:
e = extract_exception()
cnt = 0
for i, j in zip(res, ref):
if not i == j:
break
cnt += 1
print("Break at index %d" % cnt)
raise e
def _get_tokens(self, data):
self.lexer.input(data)
return list(self.lexer)
def __init__(self, lexer=None):
if lexer is None:
self.lexer = BentoLexer(optimize=_OPTIMIZE_LEX)
else:
self.lexer = lexer
picklefile = _PICKLED_PARSETAB
if not op.exists(picklefile):
try:
fid = open(picklefile, "wb")
fid.close()
except IOError:
e = extract_exception()
raise BentoError(
"Could not write pickle file %r (original error was %r)" %
(picklefile, e))
else:
try:
fid = open(_PICKLED_PARSETAB, "wb")
fid.close()
except IOError:
# In case of read-only support (no write access, zip import,
# etc...)
e = extract_exception()
if e.errno == errno.EACCES:
if _has_parser_changed(_PICKLED_PARSETAB):
raise BentoError(
"Cannot write new updated grammar to file %r" %
_PICKLED_PARSETAB)
else:
raise
self.parser = ply.yacc.yacc(start="stmt_list",
picklefile=picklefile,
debug=_DEBUG_YACC)
def reset(self):
# XXX: implements reset for lexer
self.lexer = BentoLexer(optimize=_OPTIMIZE_LEX)
# XXX: ply parser.reset method expects those attributes to
# exist
self.parser.statestack = []
self.parser.symstack = []
return self.parser.restart()
def setUp(self):
self.lexer = BentoLexer()
def setUp(self):
self.lexer = BentoLexer()