Exemple #1
0
 def test_Tokens3(self):
     """Word Tokenize a string with an apostrophe leading a word"""
     line3 = "This is a line of 'text'ed' words"
     line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
     test3 = Language()
     Tokens = test3.w_Tokens(line3)
     assert Tokens == line_expect
Exemple #2
0
 def test_Tokens2(self):
     """Word Tokenize a string with non-word characters"""
     line2 = "This$ is# a 6 line43 of text'ed words"
     line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
     test2 = Language()
     Tokens = test2.w_Tokens(line2)
     assert Tokens == line_expect
Exemple #3
0
 def test_Tokens1(self):
     """Word Tokenize a string"""
     line1 = "This is a line of text'ed words"
     line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
     test1 = Language()
     Tokens = test1.w_Tokens(line1)
     assert Tokens == line_expect
Exemple #4
0
 def test_Tokens5(self):
     """Word Tokenize a string with apostrophne pre-pending words
     beginning with a vowel"""
     line5 = "This 'is 'a 'line of text'ed' words"
     line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
     test5 = Language()
     Tokens = test5.w_Tokens(line5)
     assert Tokens == line_expect
Exemple #5
0
 def test_Tokens4(self):
     """Word Tokenize a string with apostrophe pre-pending words 
     (including the last word in a string)"""
     line4 = "This is a line of 'text'ed' words'"
     line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
     test4 = Language()
     Tokens = test4.w_Tokens(line4)
     assert Tokens == line_expect
Exemple #6
0
from __future__ import absolute_import

from lang.lang import Language

langs = Language.langs

# Load languages
Language('en')
Language('la')