def test__get_tokens(): morph_fd = StringIO('((()))') tokens = list(nasc._get_tokens(morph_fd)) eq_(tokens, ['(', '(', '(', ')', ')', ')']) morph_fd = StringIO('(Baz("Bar"("Foo")))') tokens = list(nasc._get_tokens(morph_fd)) eq_(tokens, ['(', 'Baz', '(', '"Bar"', '(', '"Foo"', ')', ')', ')']) morph_fd = StringIO('(Baz("Cell Bar Body"("Foo")))') tokens = list(nasc._get_tokens(morph_fd)) eq_(tokens, ['(', 'Baz', '(', '"Cell Bar Body"', '(', '"Foo"', ')', ')', ')'])