def getToken():
global token
if verbose:
if token:
# print the current token, before we get the next one
# print (" "*40 ) + token.show()
print ((" " * indent) + " (" + token.show(align=False) + ")")
token = lexer.get()
def main(sourceText):
global f
f = open(outputFilename, "w")
writeln("Here are the tokens returned by the lexer:")
# create an instance of a lexer
lexer.initialize(sourceText)
#------------------------------------------------------------------
# use the lexer.getlist() method repeatedly to get the tokens in
# the sourceText. Then print the tokens.
#------------------------------------------------------------------
while True:
token = lexer.get()
writeln(token.show(True))
if token.type == EOF: break
f.close()