def test_should_accept_a_valid_token_sequence(self): source = """[A][[<-]\t\nE1x1000- <-!=-+yz100]#$""" f = StringIO(source) tokens = tokenize(f) self.assertIsNotNone(list(tokens))
def test_should_raise_SyntaxError_on_invalid_token_sequence(self): source = """->\t\n$""" with self.assertRaises(SyntaxError): list(tokenize(StringIO(source)))
def format(f): return '\n'.join(rebuild_lines(parse(tokenize(f))))