def consume(self, input:InputStream):
curChar = input.LA(1)
if curChar==ord('\n'):
self.line += 1
self.column = 0
else:
self.column += 1
input.consume()
def accept(self, input:InputStream, lexerActionExecutor:LexerActionExecutor, startIndex:int, index:int, line:int, charPos:int):
if LexerATNSimulator.debug:
print("ACTION", lexerActionExecutor)
# seek to after last char in token
input.seek(index)
self.line = line
self.column = charPos
if lexerActionExecutor is not None and self.recog is not None:
lexerActionExecutor.execute(self.recog, input, startIndex)
def match(self, input:InputStream , mode:int):
self.match_calls += 1
self.mode = mode
mark = input.mark()
try:
self.startIndex = input.index
self.prevAccept.reset()
dfa = self.decisionToDFA[mode]
if dfa.s0 is None:
return self.matchATN(input)
else:
return self.execATN(input, dfa.s0)
finally:
input.release(mark)
def evaluatePredicate(self, input:InputStream, ruleIndex:int, predIndex:int, speculative:bool):
# assume true if no recognizer was provided
if self.recog is None:
return True
if not speculative:
return self.recog.sempred(None, ruleIndex, predIndex)
savedcolumn = self.column
savedLine = self.line
index = input.index
marker = input.mark()
try:
self.consume(input)
return self.recog.sempred(None, ruleIndex, predIndex)
finally:
self.column = savedcolumn
self.line = savedLine
input.seek(index)
input.release(marker)
def accept(
self,
input: InputStream,
lexerActionExecutor: LexerActionExecutor,
startIndex: int,
index: int,
line: int,
charPos: int,
):
if self.debug:
print("ACTION %s\n", lexerActionExecutor)
# seek to after last char in token
input.seek(index)
self.line = line
self.column = charPos
if input.LA(1) != Token.EOF:
self.consume(input)
if lexerActionExecutor is not None and self.recog is not None:
lexerActionExecutor.execute(self.recog, input, startIndex)
def run_ldpath_program(self, uri, ldprogram_data):
#See https://github.com/antlr/antlr4/blob/master/runtime/Python3/src/antlr4/FileStream.py#L27
ldprogram_raw = codecs.decode(ldprogram_data, 'ascii', 'strict')
self.logger.debug("Running ldpath program on uri: {}".format(uri))
self.logger.debug("Running ldpath program: {}".format(ldprogram_data))
ldprogram = LDPathProgram(self._db, self.config["cache"]["timeout"])
# This instantiates the InputStream as needed by antlr
self.logger.debug("Creating InputStream")
ldprogram.raw = InputStream(ldprogram_raw)
self.logger.debug("Parsing")
ldprogram.parse()
self.logger.debug("Running")
ldprogram.run(uri)
return ldprogram.response
def testOverlappingReplace2(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(0, 3, 'bar')
rewriter.replaceRange(1, 2, 'foo')
with self.assertRaises(ValueError) as ctx:
rewriter.getDefaultText()
self.assertEquals(
"""replace op boundaries of <ReplaceOp@[@1,1:1='b',<2>,1:1]..[@2,2:2='c',<3>,1:2]:"foo"> overlap with previous <ReplaceOp@[@0,0:0='a',<1>,1:0]..[@3,3:2='<EOF>',<-1>,1:3]:"bar">""",
ctx.exception.message
)
def testReplaceThenReplaceLowerIndexedSuperset(self):
input = InputStream('abcccba')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(2, 4, 'xyz')
rewriter.replaceRange(1, 3, 'foo')
with self.assertRaises(ValueError) as ctx:
rewriter.getDefaultText()
msg = ctx.exception.message
self.assertEquals(
"""replace op boundaries of <ReplaceOp@[@1,1:1='b',<2>,1:1]..[@3,3:3='c',<3>,1:3]:"foo"> overlap with previous <ReplaceOp@[@2,2:2='c',<3>,1:2]..[@4,4:4='c',<3>,1:4]:"xyz">""",
msg
)
def parse(self):
if self.spec is None:
raise STLParseException('STL specification if empty')
# Parse the STL spec - ANTLR4 magic
entire_spec = self.modular_spec + self.spec
input_stream = InputStream(entire_spec)
lexer = LtlLexer(input_stream)
stream = CommonTokenStream(lexer)
parser = LtlParser(stream)
parser._listeners = [LTLParserErrorListener()]
ctx = parser.specification_file()
# Create the visitor for the actual spec nodes
visitor = LTLSpecificationParser(self)
self.top = visitor.visitSpecification_file(ctx)
def testReplaceRangeThenInsertAtRightEdge(self):
input = InputStream('abcccba')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(2, 4, 'x')
rewriter.insertBeforeIndex(4, 'y')
with self.assertRaises(ValueError) as ctx:
rewriter.getDefaultText()
msg = ctx.exception.message
self.assertEquals(
"insert op <InsertBeforeOp@[@4,4:4='c',<3>,1:4]:\"y\"> within boundaries of previous <ReplaceOp@[@2,2:2='c',<3>,1:2]..[@4,4:4='c',<3>,1:4]:\"x\">",
msg
)
def _runTest(self, text, name):
input_stream = InputStream(text)
parser = CUParser(input_stream, name)
cu = parser.parse()
if len(cu.markers) > 0:
print("Test Failed:")
in_reader = StringIO(text)
i = 1
while True:
line = in_reader.readline()
if line == "":
break
line = line[:-1]
print("%3d: %s" % (i, line))
i += 1
self.assertEqual(len(cu.markers), 0, "Errors")
def main(argv):
usingFile = False
# if argv is not None:
# inputStream = FileStream(sys.argv[1])
# usingFile = True
while True:
stdout.write("mathjunkie>> ")
stdout.flush() # this flushes the output out immediately.
if not usingFile:
inputStream = InputStream(stdin.readline())
lexer = MathCrazeLexer(inputStream)
tokenStream = CommonTokenStream(lexer)
parser = MathCrazeParser(tokenStream)
tree = parser.equal()
# lispTree = tree.toStringTree(recog=parser)
# print(lispTree)
visitor = Visitor()
print(visitor.visit(tree))
def build_objects(cdl_filename=None):
'''
Build a VesselSeason instance from the CDL file
'''
if cdl_filename is None:
input_stream = InputStream(sys.stdin.readline())
else:
input_stream = FileStream(cdl_filename)
lexer = CdlLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = CdlParser(token_stream)
tree = parser.cdl_file()
visitor = CdlFileVisitor()
cdl_file = visitor.visit(tree)
locator.save()
return cdl_file
def parse_plaintext(text, LexerClass, ParserClass, rule_name):
'''
There's half a dozen lines of boilerplate code to initialize a
lexer, parser, and parse the plaintext into a tree. This method
centralizes the logic all in one place.
Returns: the parsed tree, accessible through the rule with `rule_name`
from the parser's grammar.
'''
input_ = InputStream(text)
lexer = LexerClass(input_)
token_stream = CommonTokenStream(lexer)
parser = ParserClass(token_stream)
if hasattr(parser, rule_name):
return getattr(parser, rule_name)()
else:
raise KeyError("Main rule %s doesn't exist in your parser's grammar",
rule_name)
def context(self,
text,
symbol,
trace=False,
diag=False,
build_parse_trees=True,
as_string=False,
fail_on_error=False,
antlr_hook=None):
"""Returns an object subclass of a ``antlr4.ParserRuleContext`` corresponding to the specified symbol (possibly as a string).
Args:
text (str): the text to be parsed.
symbol (str): the symbol (rule name) the parse should start at.
trace (bool): if ``True`` the method ``antlr4.Parser.setTrace`` will be used to activate *tracing*.
diag (bool): if ``True`` the parsing will be run with a ``antlr4.error.DiagnosticErrorListener`` setting the prediction mode to ``antlr4.atn.PredictionMode.LL_EXACT_AMBIG_DETECTION``.
build_parse_trees (bool): if ``False`` the field ``antlr4.Parser.buildParseTrees`` will be set to ``False`` so that no trees will be generated.
as_string (bool): if ``True`` the method will return the string representation of the context obtained using its ``toStringTree`` method.
fail_on_error (bool): if ``True`` the method will return ``None`` in case of paring errors.
antlr_hook (function): if not ``None`` will be called with the lexer and parser as arguments to further configure them before use
Returns:
A parser context, in case of parsing errors the it can be used to investigate the errors (unless ``fail_on_error`` is ``True`` in what case this method returns ``None``).
"""
lexer = self.Lexer(InputStream(text))
stream = CommonTokenStream(lexer)
parser = self.Parser(stream)
parser.setTrace(trace)
if diag:
parser.addErrorListener(DiagnosticErrorListener())
parser._interp.predictionMode = PredictionMode.LL_EXACT_AMBIG_DETECTION
parser.buildParseTrees = build_parse_trees
if antlr_hook is not None: antlr_hook(lexer, parser)
buf = StringIO()
with redirect_stderr(buf):
ctx = getattr(parser, symbol)()
errs = buf.getvalue().strip()
if errs:
warn(errs)
if fail_on_error: return None
if as_string:
return ctx.toStringTree(recog=self.Parser)
else:
return ctx
def deductionParser(expresion):
input_stream = InputStream(expresion)
# Setup Lexer
print(f"Data:\n{input_stream}")
lexer = WangLexer(input_stream)
token_stream = CommonTokenStream(lexer)
#Setup Parser (and own ErrorListener)
parser = WangParser(token_stream)
parser.removeErrorListeners()
parser.addErrorListener(MyErrorListener())
try:
tree = parser.assertion()
except SyntaxError as e:
print(e.msg)
sys.exit(-1)
#Setup the Visitor and visit Parse tree
visitor = WangPrintVisitor()
print("*** Starts visit of data ***")
return list(visitor.visit(tree))
def compile(in_file, out_file):
global options
input = InputStream(in_file.read())
lexer = MalicLexer(input)
tokens = CommonTokenStream(lexer)
parser = MalicParser(tokens)
tree = parser.compilationUnit()
walker = ParseTreeWalker()
listener = ASTBuilder()
walker.walk(listener, tree)
ast = listener.ast
# initialize_builtin_type()
ast.resolve_symbol()
ast.check_type()
if options.enable_output_irrelevant_elimination:
ast.eliminate_output_irrelevant_node()
ir_builder = IRBuilder(ast)
ir_builder.generate_ir()
emmiter = InstructionEmmiter(ir_builder)
emmiter.emit()
cfg_builder = ControlFlowAnalyzer(emmiter)
cfg_builder.build_controlflow()
# print_instructions(emmiter.function_entities)
dataflow_analyzer = DataFlowAnalyzer(emmiter)
dataflow_analyzer.optimize()
if options.print_instruction:
print_instructions(emmiter.function_entities)
register_config = RegisterConfig()
allocator = Allocator(emmiter, register_config)
allocator.allocate()
translator = Translator(emmiter, register_config)
asm = translator.translate()
for s in asm:
out_file.write(s + '\n')
def compile(self, script, raise_exceptions=False):
input_stream = InputStream(script)
lexer = EveScriptLexer(input_stream)
if raise_exceptions:
lexer.removeErrorListeners()
lexer.addErrorListener(ThrowingErrorListener())
token_stream = CommonTokenStream(lexer)
parser = EveScriptParser(token_stream)
if raise_exceptions:
parser.removeErrorListeners()
parser.addErrorListener(ThrowingErrorListener())
tree = parser.script()
return self.visitScript(tree)
def main():
"""Funcio principal, inicialitza l'entrada, guarda en un pickle i genera una gràfica."""
outputfile = "networkx_graph"
if len(sys.argv) > 1:
input_stream = FileStream(sys.argv[1], encoding="utf-8")
if len(sys.argv) > 2:
outputfile = sys.argv[2]
else:
input_stream = InputStream(input("? "))
graph = parse_to_network(input_stream)
graph_file = outputfile + ".pckl"
plot_file = outputfile + ".png"
save_pickle(graph, graph_file)
graph_restored = read_pickle(graph_file)
plot_network(graph_restored, plot_file)
def testToStringStartStop2(self):
input = InputStream('x = 3 * 0 + 2 * 0;')
lexer = TestLexer2(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
self.assertEquals('x = 3 * 0 + 2 * 0;', rewriter.getDefaultText())
# replace 3 * 0 with 0
rewriter.replaceRange(4, 8, '0')
self.assertEquals('x = 0 + 2 * 0;', rewriter.getDefaultText())
self.assertEquals('x = 0 + 2 * 0;', rewriter.getText('default', Interval(0, 17)))
self.assertEquals('0', rewriter.getText('default', Interval(4, 8)))
self.assertEquals('x = 0', rewriter.getText('default', Interval(0, 8)))
self.assertEquals('2 * 0', rewriter.getText('default', Interval(12, 16)))
rewriter.insertAfter(17, "// comment")
self.assertEquals('2 * 0;// comment', rewriter.getText('default', Interval(12, 18)))
self.assertEquals('x = 0', rewriter.getText('default', Interval(0, 8)))
def test_basic_operation(self):
not_a_target = """# Not a target:
.l.r:
# Builtin rule
# Implicit rule search has not been done.
# Modification time never checked.
# File has not been updated.
# recipe to execute (built-in):
$(LEX.l) $< > $@
mv -f lex.yy.r $@"""
phony = """.PHONY: all
# Implicit rule search has not been done.
# Modification time never checked.
# File has not been updated."""
target1 = """objdir/bar.o: bar.c | objdir
# Implicit rule search has been done.
# Implicit/static pattern stem: 'bar'
# Last modified 2017-02-20 14:12:50.825407391
# File has been updated.
# Successfully updated.
# recipe to execute (from 'Makefile', line 5):
touch $@"""
target2 = """all: objdir/foo.o objdir/bar.o objdir/baz.o
# Phony target (prerequisite of .PHONY).
# Implicit rule search has not been done.
# File does not exist.
# File has been updated.
# Successfully updated."""
string = '\n\n'.join([not_a_target, phony, target1, not_a_target, target2])
input_ = InputStream(string)
lexer = TargetParagraphLexer(input_)
token = lexer.nextToken()
self.assertEqual(token.text, ''.join([target1, '\n']))
self.assertEqual(token.type, TargetParagraphLexer.TARGET_PARAGRAPH)
token = lexer.nextToken()
self.assertEqual(token.text, target2)
self.assertEqual(token.type, TargetParagraphLexer.TARGET_PARAGRAPH)
token = lexer.nextToken()
self.assertEqual(token.text, '<EOF>')
self.assertEqual(token.type, Token.EOF)
def main():
global _verbosity
compile_units = []
parser = get_parser()
argv = []
i = 1
while i < len(sys.argv):
arg = sys.argv[i]
argv.append(arg)
i += 1
args = parser.parse_args(argv)
if args.v:
_verbosity = 1
for f in args.files:
if not os.path.isfile(f):
print("Error: file \"" + f + "\" does not exist")
sys.exit(1)
with open(f, "r") as fp:
input_stream = InputStream(fp.read())
verbose("Parsing file %s", f)
parser = CUParser(input_stream, f)
cu = parser.parse()
if len(cu.markers) > 0:
for m in cu.markers:
print("Error: " + m.msg)
sys.exit(1)
compile_units.append(cu)
if args.link:
print("Error: link not currently supported")
sys.exit(1)
def execute(self, lexer:Lexer, input:InputStream, startIndex:int):
requiresSeek = False
stopIndex = input.index
try:
for lexerAction in self.lexerActions:
if isinstance(lexerAction, LexerIndexedCustomAction):
offset = lexerAction.offset
input.seek(startIndex + offset)
lexerAction = lexerAction.action
requiresSeek = (startIndex + offset) != stopIndex
elif lexerAction.isPositionDependent:
input.seek(stopIndex)
requiresSeek = False
lexerAction.execute(lexer)
finally:
if requiresSeek:
input.seek(stopIndex)
def execute(self, lexer: Lexer, input: InputStream, startIndex: int):
requiresSeek = False
stopIndex = input.index
try:
for lexerAction in self.lexerActions:
if isinstance(lexerAction, LexerIndexedCustomAction):
offset = lexerAction.offset
input.seek(startIndex + offset)
lexerAction = lexerAction.action
requiresSeek = (startIndex + offset) != stopIndex
elif lexerAction.isPositionDependent:
input.seek(stopIndex)
requiresSeek = False
lexerAction.execute(lexer)
finally:
if requiresSeek:
input.seek(stopIndex)
def evaluacion(to_parse_line):
input_stream = InputStream(to_parse_line)
# Setup Lexer
print(f"Data:\n{input_stream}")
lexer = WangLexer(input_stream)
token_stream = CommonTokenStream(lexer)
#Setup Parser (and own ErrorListener)
parser = WangParser(token_stream)
parser.removeErrorListeners()
parser.addErrorListener(MyErrorListener())
try:
tree = parser.assertion()
except SyntaxError as e:
print(e.msg)
sys.exit(-1)
#Setup the Visitor and visit Parse tree
visitor = WangPrintVisitor()
print("*** Starts visit of data ***")
res = visitor.visit(tree)
print(f"*** La evaluacion de su propuesta es igual a {res} ***")
return f"*** La evaluacion de su propuesta es igual a {res} ***"
def parse(answerSets, clingoOutput, two_stageParsing):
tokens = CommonTokenStream(ClingoLexer(InputStream(clingoOutput)))
parser = ClingoParser(tokens)
visitor = ClingoParserVisitorImplementation(answerSets)
if not two_stageParsing:
visitor.visit(parser.output())
return
parser._interp.predictionMode = PredictionMode.SLL
parser.removeErrorListeners()
parser._errHandler = BailErrorStrategy()
try:
visitor.visit(parser.output())
except RuntimeError as exception:
if isinstance(exception, RecognitionException):
tokens.seek(0)
parser.addErrorListener(ConsoleErrorListener.INSTANCE)
parser._errHandler = DefaultErrorStrategy()
parser._interp.predictionMode = PredictionMode.LL
visitor.visit(parser.output())
def parse(atom):
tokens = CommonTokenStream(ASPGrammarLexer(InputStream(atom)))
parser = ASPGrammarParser(tokens)
visitor = ASPParser()
parser._interp.predictionMode = PredictionMode.SLL
parser.removeErrorListeners()
parser._errHandler = BailErrorStrategy()
try:
visitor.visit(parser.output())
except RuntimeError as exception:
if isinstance(exception, RecognitionException):
tokens.seek(0)
parser.addErrorListener(ConsoleErrorListener.INSTANCE)
parser._errHandler = DefaultErrorStrategy()
parser._interp.predictionMode = PredictionMode.LL
visitor.visit(parser.output())
return visitor
def interactive():
Plotter.interactive(True)
parser = GraphParser(None)
visitor = ExecVisitor()
lineno = 1
while True:
line = input(f"Graph [{lineno}]> ")
istream = InputStream(line)
lexer = GraphLexer(istream)
lexer.line = lineno
lexer.column = 0
token_stream = CommonTokenStream(lexer)
parser.setInputStream(token_stream)
tree = parser.statement()
result = visitor.visit(tree)
print(f"Out [{lineno}]>", result, '\n')
lineno += 1
def readAndPrint(str_stream):
input_stream = InputStream(str_stream)
lexer = reactLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = reactParser(token_stream)
#tree = parser.reaction()
tree = parser.entries()
#print(tree.toStringTree(recog=parser))
visitor = ReactVisitor()
visitor.visit(tree)
print()
visitor.genElements()
print()
visitor.printSODE()
print()
visitor.printJacobian()
print()
from stlgrammarLexer import stlgrammarLexer
from stlgrammarParser import stlgrammarParser
from stlgrammarInterpreter import stlgrammarInterpreter
from stlgrammarSimplifier import stlgrammarSimplifier
if __name__ == '__main__':
if len(sys.argv) > 1:
stlFile = open(sys.argv[1], 'r')
else:
# input_stream = InputStream(sys.stdin.readline())
stlFile = open('stl.expr', 'r')
for line in stlFile.readlines():
stlRule = line.strip()
lexer = stlgrammarLexer(InputStream(stlRule))
token_stream = CommonTokenStream(lexer)
parser = stlgrammarParser(token_stream)
tree = parser.prog() # parse; start at prog
# print tree as text
lisp_tree_str = tree.toStringTree(recog=parser)
print(lisp_tree_str)
# stlgrammarSimplifier
simplifier = stlgrammarSimplifier()
walker = ParseTreeWalker()
walker.walk(simplifier, tree)
# stlgrammarInterpreter
def enterComparatorExpression(
self, ctx: SimpleBooleanParser.ComparatorExpressionContext):
print('comparation expr')
def enterIdentifierExpression(
self, ctx: SimpleBooleanParser.IdentifierExpressionContext):
print('identifier')
if __name__ == '__main__':
variables = {
'a',
True,
'A',
True,
}
exprs = ['A > 2' '1 >= 1.0']
for expr in exprs:
expr_stream = InputStream(expr)
lexer = SimpleBooleanLexer(expr_stream)
stream = CommonTokenStream(lexer)
parser = SimpleBooleanParser(stream)
tree = parser.parse()
#printer = CustomVisitor()
#printer.visit(tree)
walker = ParseTreeWalker()
walker.walk(CustomListener(), tree)
def getText(self, input: InputStream):
# index is first lookahead char, don't include.
return input.getText(self.startIndex, input.index - 1)
def __init__(self):
InputStream.__init__(self, sys.stdin.read())
def newTokenStreamFromString(self, input_):
stream = InputStream(input_)
return EIndentingLexer(stream)
if __name__ == '__main__':
print("*** Testing Wang Parser (EIF400 II-2018) ***")
if len(sys.argv) > 1:
file = sys.argv[1]
print(f'*** Processing from file "{file}" ***')
input_stream = FileStream(file)
else:
print(f'*** Processing from console ***\n>', end='')
to_parse_line = sys.stdin.readline()
# Use demo line if none was typed
if len(to_parse_line) <= 1: # Just enter was hit
to_parse_line = "q1 | q2 & q3 => q1, q2"
print(f"Empty line. Testing with demo: {to_parse_line}")
#
input_stream = InputStream(to_parse_line)
# Setup Lexer
print(f"Data:\n{input_stream}")
lexer = WangLexer(input_stream)
token_stream = CommonTokenStream(lexer)
#Setup Parser (and own ErrorListener)
parser = WangParser(token_stream)
parser.removeErrorListeners()
parser.addErrorListener(MyErrorListener())
try:
tree = parser.assertion()
except SyntaxError as e:
print(e.msg)
sys.exit(-1)
#Setup the Visitor and visit Parse tree
def getText(self, input:InputStream):
# index is first lookahead char, don't include.
return input.getText(self.startIndex, input.index-1)
for function_name, message_text in zip(self.function_list, self.message_list):
test_function_name = function_name.replace("(", "(self, ")
unit_test_string += "def test_" + test_function_name + ":\n\t\tself.assertFalse("
unit_test_string += "" + function_name + ", '" + message_text + "')\n\n\t"
unit_test_string += "\n\nif __name__ == \"__main__\":\n\tif len(sys.argv) > 1:\n\t\t"
unit_test_string += "potential_file = sys.argv.pop()\n\t\tif os.path.isfile(potential_file):\n\t\t\t"
unit_test_string += "with open(potential_file, 'r') as read_file:\n\t\t\t\tMistakeTest.SUBMISSION_STRING = "
unit_test_string += "read_file.read()\n\tunittest.main()"
return unit_test_string
if __name__ == '__main__':
if len(sys.argv) > 1: # len(sys.argv) > 2 means output_file then input_file
input_stream = FileStream(sys.argv[1])
else:
input_stream = InputStream(sys.stdin.read())
lexer = specLang_draftP4Lexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = specLang_draftP4Parser(token_stream)
tree = parser.document()
#lisp_tree_str = tree.toStringTree(recog=parser)
#print(lisp_tree_str)
# listener
print("Start Walking...")
listener = SpecLangToPedalCodeTranslator()
walker = ParseTreeWalker()
walker.walk(listener, tree)
final_string = listener.getTranslationString()
def tokenize(self, word: str) -> Tuple[Token, ...]:
return self.get_all_tokens(self.lexer_instance(InputStream(word)))