def main (): """main() Tokenize an input file or stdin, if no file given in script arguments. """ from TokenUtils import testTokenizer testTokenizer(PyTokenizer)
def main (): """main() Run a silly little test on the tokenizer class. """ from TokenUtils import testTokenizer testTokenizer(StdTokenizer)
def main (): """main() Run the tokenizer on stdin input, printing any tokens that are recognized. """ from TokenUtils import testTokenizer testTokenizer(StdTokenizer)