def test_empty_list_on_empty_data(self):
"""Test Tokenizer works on empty list"""
tokenizer = AntlrTokenizer(JavaScriptLexer)
data = ""
tokens = tokenizer.tokenize(data)
self.assertEqual(tokens, [])
def test_tokenizes_malformed_without_error(self):
"""Tests tokenizer doesnt error on garbage input"""
tokenizer = AntlrTokenizer(JavaScriptLexer)
txt = "aasdfj1 1jhsdf9 1 3@ 1 + => adj 193"
tokens = tokenizer.tokenize(txt)
self.assertEqual(tokens, [
'aasdfj1', ' ', '1', 'jhsdf9', ' ', '1', ' ', '3', '@', ' ', '1',
' ', '+', ' ', '=>', ' ', 'adj', ' ', '193'
])
def test_combine_same_as_orig(self):
"""Tests the token combiner"""
tokenizer = AntlrTokenizer(JavaScriptLexer)
txt = """async function process(array) {
for await (let i of array) {
doSomething(i);
}
}"""
tokens = tokenizer.tokenize(txt)
self.assertEqual(tokenizer.combine(tokens), txt)
def test_tokenize_simple_js_file(self):
"""Test tokenizer works with sample JS"""
tokenizer = AntlrTokenizer(JavaScriptLexer)
txt = """async function process(array) {
for await (let i of array) {
doSomething(i);
}
}"""
tokens = tokenizer.tokenize(txt)
self.assertEqual(tokens, [
'async', ' ', 'function', ' ', 'process', '(', 'array', ')', ' ',
'{', '\n', ' ', 'for', ' ', 'await', ' ', '(', 'let', ' ',
'i', ' ', 'of', ' ', 'array', ')', ' ', '{', '\n',
' ', 'doSomething', '(', 'i', ')', ';', '\n',
' ', '}', '\n', ' ', '}'
])
