def testParseInput(self):
lexer = ZLexer(self.mem)
results = lexer.parse_input(phrase3)
assert results == [['the', 40278], ['fish', 33150], [',', 0], \
['house', 34572], [',', 0], ['and', 29874], \
['bird', 30576], ['are', 0], ['odd', 36525], \
['and', 29874], ['round', 38361], [',', 0], \
['no', 36300]]
def testParseInput(self):
lexer = ZLexer(self.mem)
results = lexer.parse_input(phrase3)
assert results == [[u'the', 40278], [u'fish', 33150], [u',', 0], \
[u'house', 34572], [u',', 0], [u'and', 29874], \
[u'bird', 30576], [u'are', 0], [u'odd', 36525], \
[u'and', 29874], [u'round', 38361], [u',', 0], \
[u'no', 36300]]
def testTokenisation(self):
lexer = ZLexer(self.mem)
separators = (',', ';', '.')
tokens = lexer._tokenise_string(phrase1, separators)
assert tokens == ['the', 'quick', 'brown', 'fox']
tokens = lexer._tokenise_string(phrase2, separators)
assert tokens == ['the', 'quick', ',', 'brown', ',', ';', 'fox', \
'might', ',', ',', 'be', 'fe', ';', 'el', '.', 'ing', \
';', ';', 'odd', ',', '.', 'today']
def testTokenisation(self):
lexer = ZLexer(self.mem)
separators = (u',', u';', u'.')
tokens = lexer._tokenise_string(phrase1, separators)
assert tokens == ['the', 'quick', 'brown', 'fox']
tokens = lexer._tokenise_string(phrase2, separators)
assert tokens == ['the', 'quick', ',', 'brown', ',', ';', 'fox', \
'might', ',', ',', 'be', 'fe', ';', 'el', '.', 'ing', \
';', ';', 'odd', ',', '.', 'today']
def testParseDictionary(self):
# load game's 'standard' dictionary into a python dictionary
lexer = ZLexer(self.mem)
num_keys = len(list(lexer._dict.keys()))
assert num_keys == lexer._num_entries, \
"lexer didn't parse all dictionary entries"
