def setUp(self):
self.lexer = BSTLexer()
def test_basic_bst():
lexer = BSTLexer()
data = """
% BibTeX standard bibliography style `plain'
INTEGERS { output.state before.all }
FUNCTION {sort.format.title}
{ 't :=
"A " #2
"An " #3
"The " #4 t chop.word
chop.word
chop.word
sortify
#1 global.max$ substring$
}
ITERATE {call.type$}
"""
tokens = [
(Token.Comment.SingleLine,
"% BibTeX standard bibliography style `plain'"),
(Token.Text, u'\n\n'),
(Token.Keyword, u'INTEGERS'),
(Token.Text, u' '),
(Token.Punctuation, u'{'),
(Token.Text, u' '),
(Token.Name.Variable, u'output.state'),
(Token.Text, u' '),
(Token.Name.Variable, u'before.all'),
(Token.Text, u' '),
(Token.Punctuation, u'}'),
(Token.Text, u'\n\n'),
(Token.Keyword, u'FUNCTION'),
(Token.Text, u' '),
(Token.Punctuation, u'{'),
(Token.Name.Variable, u'sort.format.title'),
(Token.Punctuation, u'}'),
(Token.Text, u'\n'),
(Token.Punctuation, u'{'),
(Token.Text, u' '),
(Token.Name.Function, u"'t"),
(Token.Text, u' '),
(Token.Name.Variable, u':='),
(Token.Text, u'\n'),
(Token.Literal.String, u'"A "'),
(Token.Text, u' '),
(Token.Literal.Number, u'#2'),
(Token.Text, u'\n '),
(Token.Literal.String, u'"An "'),
(Token.Text, u' '),
(Token.Literal.Number, u'#3'),
(Token.Text, u'\n '),
(Token.Literal.String, u'"The "'),
(Token.Text, u' '),
(Token.Literal.Number, u'#4'),
(Token.Text, u' '),
(Token.Name.Variable, u't'),
(Token.Text, u' '),
(Token.Name.Variable, u'chop.word'),
(Token.Text, u'\n '),
(Token.Name.Variable, u'chop.word'),
(Token.Text, u'\n'),
(Token.Name.Variable, u'chop.word'),
(Token.Text, u'\n'),
(Token.Name.Variable, u'sortify'),
(Token.Text, u'\n'),
(Token.Literal.Number, u'#1'),
(Token.Text, u' '),
(Token.Name.Builtin, u'global.max$'),
(Token.Text, u' '),
(Token.Name.Builtin, u'substring$'),
(Token.Text, u'\n'),
(Token.Punctuation, u'}'),
(Token.Text, u'\n\n'),
(Token.Keyword, u'ITERATE'),
(Token.Text, u' '),
(Token.Punctuation, u'{'),
(Token.Name.Builtin, u'call.type$'),
(Token.Punctuation, u'}'),
(Token.Text, u'\n'),
]
assert list(lexer.get_tokens(textwrap.dedent(data))) == tokens