def test_Tokens2(self):
"""Word Tokenize a string with non-word characters"""
line2 = "This$ is# a 6 line43 of text'ed words"
line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
test2 = Language()
Tokens = test2.w_Tokens(line2)
assert Tokens == line_expect
def test_Tokens3(self):
"""Word Tokenize a string with an apostrophe leading a word"""
line3 = "This is a line of 'text'ed' words"
line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
test3 = Language()
Tokens = test3.w_Tokens(line3)
assert Tokens == line_expect
def test_Tokens3(self):
"""Word Tokenize a string with an apostrophe leading a word"""
line3 = "This is a line of 'text'ed' words"
line_expect = ["This","is","a","line","of",u"text'ed","words"]
test3 = Language()
Tokens = test3.w_Tokens(line3)
assert Tokens == line_expect
def test_Tokens1(self):
"""Word Tokenize a string"""
line1 = "This is a line of text'ed words"
line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
test1 = Language()
Tokens = test1.w_Tokens(line1)
assert Tokens == line_expect
def test_Tokens2(self):
"""Word Tokenize a string with non-word characters"""
line2 = "This$ is# a 6 line43 of text'ed words"
line_expect = ["This","is","a","line","of",u"text'ed","words"]
test2 = Language()
Tokens = test2.w_Tokens(line2)
assert Tokens == line_expect
def test_Tokens1(self):
"""Word Tokenize a string"""
line1 = "This is a line of text'ed words"
line_expect = ["This","is","a","line","of",u"text'ed", "words"]
test1 = Language()
Tokens = test1.w_Tokens(line1)
assert Tokens == line_expect
def test_Tokens5(self):
"""Word Tokenize a string with apostrophne pre-pending words
beginning with a vowel"""
line5 = "This 'is 'a 'line of text'ed' words"
line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
test5 = Language()
Tokens = test5.w_Tokens(line5)
assert Tokens == line_expect
def test_Tokens4(self):
"""Word Tokenize a string with apostrophe pre-pending words
(including the last word in a string)"""
line4 = "This is a line of 'text'ed' words'"
line_expect = ["This", "is", "a", "line", "of", u"text'ed", "words"]
test4 = Language()
Tokens = test4.w_Tokens(line4)
assert Tokens == line_expect
def test_Tokens5(self):
"""Word Tokenize a string with apostrophne pre-pending words
beginning with a vowel"""
line5 = "This 'is 'a 'line of text'ed' words"
line_expect = ["This","is","a","line","of",u"text'ed","words"]
test5 = Language()
Tokens = test5.w_Tokens(line5)
assert Tokens == line_expect
def test_Tokens4(self):
"""Word Tokenize a string with apostrophe pre-pending words
(including the last word in a string)"""
line4 = "This is a line of 'text'ed' words'"
line_expect = ["This","is","a","line","of",u"text'ed","words"]
test4 = Language()
Tokens = test4.w_Tokens(line4)
assert Tokens == line_expect
from __future__ import absolute_import
from lang.lang import Language
langs = Language.langs
# Load languages
Language('en')
Language('la')