def test_spaces_fr(self):
self.assertEqual(normalize(" c'est le test", lang="fr-fr"),
"c'est test")
self.assertEqual(normalize(" c'est le test ", lang="fr-fr"),
"c'est test")
self.assertEqual(normalize(" c'est un test", lang="fr-fr"),
"c'est 1 test")
def test_articles_es(self):
self.assertEqual(
normalize("esta es la prueba", lang="es", remove_articles=True),
"esta es prueba")
self.assertEqual(
normalize("y otra prueba", lang="es", remove_articles=True),
"y otra prueba")
def test_spaces_pt(self):
self.assertEqual(normalize(" isto e o teste", lang="pt"),
"isto teste")
self.assertEqual(normalize(" isto sao os testes ", lang="pt"),
"isto sao testes")
self.assertEqual(normalize(" isto e um teste", lang="pt",
remove_articles=False),
"isto e 1 teste")
def test_agressive_pruning_pt(self):
self.assertEqual(normalize("uma palavra", lang="pt"),
"1 palavra")
self.assertEqual(normalize("esta palavra um", lang="pt"),
"palavra 1")
self.assertEqual(normalize("o homem batia-lhe", lang="pt"),
"homem batia")
self.assertEqual(normalize("quem disse asneira nesse dia", lang="pt"),
"quem disse asneira dia")
def test_numbers(self):
self.assertEqual(normalize("dette er en to tre test", lang="da-dk"),
"dette er 1 2 3 test")
self.assertEqual(
normalize("dette er fire fem seks test", lang="da-dk"),
"dette er 4 5 6 test")
self.assertEqual(normalize("dette er syv otte ni test", lang="da-dk"),
"dette er 7 8 9 test")
self.assertEqual(normalize("dette er ti elve tolv test", lang="da-dk"),
"dette er 10 11 12 test")
def test_articles(self):
self.assertEqual(normalize("this is a test", remove_articles=True),
"this is test")
self.assertEqual(normalize("this is the test", remove_articles=True),
"this is test")
self.assertEqual(normalize("and another test", remove_articles=True),
"and another test")
self.assertEqual(
normalize("this is an extra test", remove_articles=False),
"this is an extra test")
def test_articles(self):
self.assertEqual(
normalize("dies ist der test", lang="de-de", remove_articles=True),
"dies ist test")
self.assertEqual(
normalize("und noch ein Test", lang="de-de", remove_articles=True),
"und noch 1 Test")
self.assertEqual(normalize("dies ist der Extra-Test", lang="de-de",
remove_articles=False),
"dies ist der Extra-Test")
def test_articles(self):
self.assertEqual(
normalize("dette er en test", lang="da-dk", remove_articles=True),
"dette er 1 test")
self.assertEqual(
normalize("og endnu en test", lang="da-dk", remove_articles=True),
"og endnu 1 test")
self.assertEqual(
normalize("dette er en extra-test",
lang="da-dk",
remove_articles=False), "dette er 1 extra-test")
def test_articles_fr(self):
self.assertEqual(normalize("c'est le test", remove_articles=True,
lang="fr-fr"),
"c'est test")
self.assertEqual(normalize("et l'autre test", remove_articles=True,
lang="fr-fr"),
"et autre test")
self.assertEqual(normalize("et la tentative", remove_articles=True,
lang="fr-fr"),
"et tentative")
self.assertEqual(normalize("la dernière tentative",
remove_articles=False, lang="fr-fr"),
"la dernière tentative")
def test_numbers(self):
self.assertEqual(
normalize("dies ist eins zwei drei test", lang="de-de"),
"dies ist 1 2 3 test")
self.assertEqual(
normalize(u"es ist vier fünf sechs test", lang="de-de"),
"es ist 4 5 6 test")
self.assertEqual(
normalize("es ist sieben acht neun test", lang="de-de"),
"es ist 7 8 9 test")
self.assertEqual(
normalize("es ist sieben acht neun test", lang="de-de"),
"es ist 7 8 9 test")
self.assertEqual(
normalize(u"dies ist zehn elf zwölf test", lang="de-de"),
"dies ist 10 11 12 test")
self.assertEqual(
normalize("dies ist dreizehn vierzehn test", lang="de-de"),
"dies ist 13 14 test")
self.assertEqual(
normalize(u"dies ist fünfzehn sechzehn siebzehn", lang="de-de"),
"dies ist 15 16 17")
self.assertEqual(
normalize("dies ist achtzehn neunzehn zwanzig", lang="de-de"),
"dies ist 18 19 20")
def test_articles_pt(self):
self.assertEqual(normalize(u"isto é o teste",
lang="pt", remove_articles=True),
u"isto teste")
self.assertEqual(
normalize(u"isto é a frase", lang="pt", remove_articles=True),
u"isto frase")
self.assertEqual(
normalize("e outro teste", lang="pt", remove_articles=True),
"outro teste")
self.assertEqual(normalize(u"isto é o teste extra",
lang="pt",
remove_articles=False), u"isto e o teste"
u" extra")
def test_numbers_pt(self):
self.assertEqual(normalize(u"isto e o um dois três teste", lang="pt"),
u"isto 1 2 3 teste")
self.assertEqual(normalize(u"ê a sete oito nove test", lang="pt"),
u"7 8 9 test")
self.assertEqual(
normalize("teste zero dez onze doze treze", lang="pt"),
"teste 0 10 11 12 13")
self.assertEqual(
normalize("teste mil seiscentos e sessenta e seis", lang="pt",
remove_articles=False),
"teste 1000 600 e 66")
self.assertEqual(
normalize("teste sete e meio", lang="pt",
remove_articles=False),
"teste 7 e meio")
self.assertEqual(
normalize("teste dois ponto nove", lang="pt"),
"teste 2 ponto 9")
self.assertEqual(
normalize("teste cento e nove", lang="pt",
remove_articles=False),
"teste 100 e 9")
self.assertEqual(
normalize("teste vinte e 1", lang="pt"),
"teste 20 1")
def test_spaces_it(self):
"""
Test cases for Italian remove spaces
"""
self.assertEqual(normalize('questo è un test ', lang='it'),
'questo è 1 test')
self.assertEqual(normalize('un altro test ', lang='it'),
'1 altro test')
self.assertEqual(
normalize('questa è un\' altra amica ',
lang='it',
remove_articles=False), 'questa è 1 altra amica')
self.assertEqual(
normalize('questo è un test ',
lang='it',
remove_articles=False), 'questo è 1 test')
def test_articles_it(self):
"""
Test cases for Italian remove_articles
"""
self.assertEqual(
normalize('questo è il test', lang='it', remove_articles=True),
'questo è test')
self.assertEqual(
normalize('questa è la frase', lang='it', remove_articles=True),
'questa è frase')
self.assertEqual(
normalize('questo è lo scopo', lang='it', remove_articles=True),
'questo è scopo')
self.assertEqual(
normalize('questo è il test extra',
lang='it',
remove_articles=False), 'questo è il test extra')
def test_numbers_it(self):
"""
Test cases for Italian normalize lang='it'
"""
self.assertEqual(normalize('è un test sette otto nove', lang='it'),
'è 1 test 7 8 9')
self.assertEqual(
normalize('test zero dieci undici dodici tredici', lang='it'),
'test 0 10 11 12 13')
self.assertEqual(
normalize('test mille seicento sessanta e sei',
lang='it',
remove_articles=False), 'test 1000 600 60 e 6')
self.assertEqual(
normalize('test sette e mezzo', lang='it', remove_articles=False),
'test 7 e 0.5')
self.assertEqual(normalize('test due punto nove', lang='it'),
'test 2 punto 9')
self.assertEqual(
normalize('test cento e nove', lang='it', remove_articles=False),
'test 100 e 9')
self.assertEqual(normalize('test venti e 1', lang='it'), 'test 20 e 1')
self.assertEqual(normalize('test ventuno e ventisette', lang='it'),
'test 21 e 27')
def test_contractions(self):
self.assertEqual(normalize("ain't"), "is not")
self.assertEqual(normalize("aren't"), "are not")
self.assertEqual(normalize("can't"), "can not")
self.assertEqual(normalize("could've"), "could have")
self.assertEqual(normalize("couldn't"), "could not")
self.assertEqual(normalize("didn't"), "did not")
self.assertEqual(normalize("doesn't"), "does not")
self.assertEqual(normalize("don't"), "do not")
self.assertEqual(normalize("gonna"), "going to")
self.assertEqual(normalize("gotta"), "got to")
self.assertEqual(normalize("hadn't"), "had not")
self.assertEqual(normalize("hadn't have"), "had not have")
self.assertEqual(normalize("hasn't"), "has not")
self.assertEqual(normalize("haven't"), "have not")
# TODO: Ambiguous with "he had"
self.assertEqual(normalize("he'd"), "he would")
self.assertEqual(normalize("he'll"), "he will")
# TODO: Ambiguous with "he has"
self.assertEqual(normalize("he's"), "he is")
# TODO: Ambiguous with "how would"
self.assertEqual(normalize("how'd"), "how did")
self.assertEqual(normalize("how'll"), "how will")
# TODO: Ambiguous with "how has" and "how does"
self.assertEqual(normalize("how's"), "how is")
# TODO: Ambiguous with "I had"
self.assertEqual(normalize("I'd"), "I would")
self.assertEqual(normalize("I'll"), "I will")
self.assertEqual(normalize("I'm"), "I am")
self.assertEqual(normalize("I've"), "I have")
self.assertEqual(normalize("I haven't"), "I have not")
self.assertEqual(normalize("isn't"), "is not")
self.assertEqual(normalize("it'd"), "it would")
self.assertEqual(normalize("it'll"), "it will")
# TODO: Ambiguous with "it has"
self.assertEqual(normalize("it's"), "it is")
self.assertEqual(normalize("it isn't"), "it is not")
self.assertEqual(normalize("mightn't"), "might not")
self.assertEqual(normalize("might've"), "might have")
self.assertEqual(normalize("mustn't"), "must not")
self.assertEqual(normalize("mustn't have"), "must not have")
self.assertEqual(normalize("must've"), "must have")
self.assertEqual(normalize("needn't"), "need not")
self.assertEqual(normalize("oughtn't"), "ought not")
self.assertEqual(normalize("shan't"), "shall not")
# TODO: Ambiguous wiht "she had"
self.assertEqual(normalize("she'd"), "she would")
self.assertEqual(normalize("she hadn't"), "she had not")
self.assertEqual(normalize("she'll"), "she will")
self.assertEqual(normalize("she's"), "she is")
self.assertEqual(normalize("she isn't"), "she is not")
self.assertEqual(normalize("should've"), "should have")
self.assertEqual(normalize("shouldn't"), "should not")
self.assertEqual(normalize("shouldn't have"), "should not have")
self.assertEqual(normalize("somebody's"), "somebody is")
# TODO: Ambiguous with "someone had"
self.assertEqual(normalize("someone'd"), "someone would")
self.assertEqual(normalize("someone hadn't"), "someone had not")
self.assertEqual(normalize("someone'll"), "someone will")
# TODO: Ambiguous with "someone has"
self.assertEqual(normalize("someone's"), "someone is")
self.assertEqual(normalize("that'll"), "that will")
# TODO: Ambiguous with "that has"
self.assertEqual(normalize("that's"), "that is")
# TODO: Ambiguous with "that had"
self.assertEqual(normalize("that'd"), "that would")
# TODO: Ambiguous with "there had"
self.assertEqual(normalize("there'd"), "there would")
self.assertEqual(normalize("there're"), "there are")
# TODO: Ambiguous with "there has"
self.assertEqual(normalize("there's"), "there is")
# TODO: Ambiguous with "they had"
self.assertEqual(normalize("they'd"), "they would")
self.assertEqual(normalize("they'll"), "they will")
self.assertEqual(normalize("they won't have"), "they will not have")
self.assertEqual(normalize("they're"), "they are")
self.assertEqual(normalize("they've"), "they have")
self.assertEqual(normalize("they haven't"), "they have not")
self.assertEqual(normalize("wasn't"), "was not")
# TODO: Ambiguous wiht "we had"
self.assertEqual(normalize("we'd"), "we would")
self.assertEqual(normalize("we would've"), "we would have")
self.assertEqual(normalize("we wouldn't"), "we would not")
self.assertEqual(normalize("we wouldn't have"), "we would not have")
self.assertEqual(normalize("we'll"), "we will")
self.assertEqual(normalize("we won't have"), "we will not have")
self.assertEqual(normalize("we're"), "we are")
self.assertEqual(normalize("we've"), "we have")
self.assertEqual(normalize("weren't"), "were not")
self.assertEqual(normalize("what'd"), "what did")
self.assertEqual(normalize("what'll"), "what will")
self.assertEqual(normalize("what're"), "what are")
# TODO: Ambiguous with "what has" / "what does")
self.assertEqual(normalize("whats"), "what is")
self.assertEqual(normalize("what's"), "what is")
self.assertEqual(normalize("what've"), "what have")
# TODO: Ambiguous with "when has"
self.assertEqual(normalize("when's"), "when is")
self.assertEqual(normalize("where'd"), "where did")
# TODO: Ambiguous with "where has" / where does"
self.assertEqual(normalize("where's"), "where is")
self.assertEqual(normalize("where've"), "where have")
# TODO: Ambiguous with "who had" "who did")
self.assertEqual(normalize("who'd"), "who would")
self.assertEqual(normalize("who'd've"), "who would have")
self.assertEqual(normalize("who'll"), "who will")
self.assertEqual(normalize("who're"), "who are")
# TODO: Ambiguous with "who has" / "who does"
self.assertEqual(normalize("who's"), "who is")
self.assertEqual(normalize("who've"), "who have")
self.assertEqual(normalize("why'd"), "why did")
self.assertEqual(normalize("why're"), "why are")
# TODO: Ambiguous with "why has" / "why does"
self.assertEqual(normalize("why's"), "why is")
self.assertEqual(normalize("won't"), "will not")
self.assertEqual(normalize("won't've"), "will not have")
self.assertEqual(normalize("would've"), "would have")
self.assertEqual(normalize("wouldn't"), "would not")
self.assertEqual(normalize("wouldn't've"), "would not have")
self.assertEqual(normalize("ya'll"), "you all")
self.assertEqual(normalize("y'all"), "you all")
self.assertEqual(normalize("y'ain't"), "you are not")
# TODO: Ambiguous with "you had"
self.assertEqual(normalize("you'd"), "you would")
self.assertEqual(normalize("you'd've"), "you would have")
self.assertEqual(normalize("you'll"), "you will")
self.assertEqual(normalize("you're"), "you are")
self.assertEqual(normalize("you aren't"), "you are not")
self.assertEqual(normalize("you've"), "you have")
self.assertEqual(normalize("you haven't"), "you have not")
def test_combinations(self):
self.assertEqual(normalize("I couldn't have guessed there'd be two"),
"I could not have guessed there would be 2")
self.assertEqual(normalize("I wouldn't have"), "I would not have")
self.assertEqual(normalize("I hadn't been there"),
"I had not been there")
self.assertEqual(normalize("I would've"), "I would have")
self.assertEqual(normalize("it hadn't"), "it had not")
self.assertEqual(normalize("it hadn't have"), "it had not have")
self.assertEqual(normalize("it would've"), "it would have")
self.assertEqual(normalize("she wouldn't have"), "she would not have")
self.assertEqual(normalize("she would've"), "she would have")
self.assertEqual(normalize("someone wouldn't have"),
"someone would not have")
self.assertEqual(normalize("someone would've"), "someone would have")
self.assertEqual(normalize("what's the weather like"),
"what is weather like")
self.assertEqual(normalize("that's what I told you"),
"that is what I told you")
self.assertEqual(normalize("whats 8 + 4"), "what is 8 + 4")
def testExtract_it(text, expected_date, expected_leftover):
res = extractWithFormat(normalize(text))
self.assertEqual(res[0], expected_date, 'per =' + text)
self.assertEqual(res[1], expected_leftover, 'per =' + text)
def testExtract(text, expected_date, expected_leftover):
res = extractWithFormat(normalize(text))
assert res[0] == expected_date
assert res[1] == expected_leftover
def test_numbers(self):
self.assertEqual(normalize("this is a one two three test"),
"this is 1 2 3 test")
self.assertEqual(normalize(" it's a four five six test"),
"it is 4 5 6 test")
self.assertEqual(normalize("it's a seven eight nine test"),
"it is 7 8 9 test")
self.assertEqual(normalize("it's a seven eight nine test"),
"it is 7 8 9 test")
self.assertEqual(normalize("that's a ten eleven twelve test"),
"that is 10 11 12 test")
self.assertEqual(normalize("that's a thirteen fourteen test"),
"that is 13 14 test")
self.assertEqual(normalize("that's fifteen sixteen seventeen"),
"that is 15 16 17")
self.assertEqual(normalize("that's eighteen nineteen twenty"),
"that is 18 19 20")
self.assertEqual(normalize("that's one nineteen twenty two"),
"that is 1 19 20 2")
self.assertEqual(normalize("that's one hundred"), "that is 1 hundred")
self.assertEqual(normalize("that's one two twenty two"),
"that is 1 2 20 2")
self.assertEqual(normalize("that's one and a half"),
"that is 1 and half")
self.assertEqual(normalize("that's one and a half and five six"),
"that is 1 and half and 5 6")
def test_spaces(self):
self.assertEqual(normalize(" dette er en test", lang="da-dk"),
"dette er 1 test")
self.assertEqual(normalize(" dette er en test ", lang="da-dk"),
"dette er 1 test")
def test_spaces(self):
self.assertEqual(normalize(" dies ist ein test", lang="de-de"),
"dies ist 1 test")
self.assertEqual(normalize(" dies ist ein test ",
lang="de-de"), "dies ist 1 test")
def test_numbers_es(self):
self.assertEqual(normalize("esto es un uno una", lang="es"),
"esto es 1 1 1")
self.assertEqual(normalize("esto es dos tres prueba", lang="es"),
"esto es 2 3 prueba")
self.assertEqual(
normalize("esto es cuatro cinco seis prueba", lang="es"),
"esto es 4 5 6 prueba")
self.assertEqual(normalize(u"siete m�s ocho m�s nueve", lang="es"),
u"7 m�s 8 m�s 9")
self.assertEqual(
normalize("diez once doce trece catorce quince", lang="es"),
"10 11 12 13 14 15")
self.assertEqual(normalize(u"dieciséis diecisiete", lang="es"),
"16 17")
self.assertEqual(normalize(u"dieciocho diecinueve", lang="es"),
"18 19")
self.assertEqual(normalize(u"veinte treinta cuarenta", lang="es"),
"20 30 40")
self.assertEqual(normalize(u"treinta y dos caballos", lang="es"),
"32 caballos")
self.assertEqual(normalize(u"cien caballos", lang="es"),
"100 caballos")
self.assertEqual(normalize(u"ciento once caballos", lang="es"),
"111 caballos")
self.assertEqual(
normalize(u"hab�a cuatrocientas una vacas", lang="es"),
u"hab�a 401 vacas")
self.assertEqual(normalize(u"dos mil", lang="es"), "2000")
self.assertEqual(
normalize(u"dos mil trescientas cuarenta y cinco", lang="es"),
"2345")
self.assertEqual(
normalize(u"ciento veintitrés mil cuatrocientas cincuenta y seis",
lang="es"), "123456")
self.assertEqual(normalize(u"quinientas veinticinco mil", lang="es"),
"525000")
self.assertEqual(
normalize(
u"novecientos noventa y nueve mil novecientos noventa y nueve",
lang="es"), "999999")
def test_numbers_fr(self):
self.assertEqual(normalize("c'est un deux trois test",
lang="fr-fr"),
"c'est 1 2 3 test")
self.assertEqual(normalize(" c'est le quatre cinq six test",
lang="fr-fr"),
"c'est 4 5 6 test")
self.assertEqual(normalize("c'est le sept huit neuf test",
lang="fr-fr"),
"c'est 7 8 9 test")
self.assertEqual(normalize("c'est le sept huit neuf test",
lang="fr-fr"),
"c'est 7 8 9 test")
self.assertEqual(normalize("voilà le test dix onze douze",
lang="fr-fr"),
"voilà test 10 11 12")
self.assertEqual(normalize("voilà le treize quatorze test",
lang="fr-fr"),
"voilà 13 14 test")
self.assertEqual(normalize("ça fait quinze seize dix-sept",
lang="fr-fr"),
"ça fait 15 16 17")
self.assertEqual(normalize("ça fait dix-huit dix-neuf vingt",
lang="fr-fr"),
"ça fait 18 19 20")
self.assertEqual(normalize("ça fait mille cinq cents",
lang="fr-fr"),
"ça fait 1500")
self.assertEqual(normalize("voilà cinq cents trente et un mille euros",
lang="fr-fr"),
"voilà 531000 euros")
self.assertEqual(normalize("voilà trois cents soixante mille cinq"
" cents quatre-vingt-dix-huit euros",
lang="fr-fr"),
"voilà 360598 euros")
self.assertEqual(normalize("voilà vingt et un euros", lang="fr-fr"),
"voilà 21 euros")
self.assertEqual(normalize("joli zéro sur vingt", lang="fr-fr"),
"joli 0 sur 20")
self.assertEqual(normalize("je veux du quatre-quart", lang="fr-fr"),
"je veux quatre-quart")
self.assertEqual(normalize("pour la neuf centième fois", lang="fr-fr"),
"pour 900e fois")
self.assertEqual(normalize("pour la première fois", lang="fr-fr"),
"pour 1er fois")
self.assertEqual(normalize("le neuf cents quatre-vingt-dix"
" millième épisode", lang="fr-fr"),
"990000e épisode")
self.assertEqual(normalize("la septième clé", lang="fr-fr"),
"7e clé")
self.assertEqual(normalize("la neuvième porte", lang="fr-fr"),
"9e porte")
self.assertEqual(normalize("le cinquième jour", lang="fr-fr"),
"5e jour")
self.assertEqual(normalize("le trois-cents-soixante-cinquième jour",
lang="fr-fr"),
"365e jour")
self.assertEqual(normalize("la 1ère fois", lang="fr-fr"),
"1er fois")
self.assertEqual(normalize("le centième centime", lang="fr-fr"),
"100e centime")
self.assertEqual(normalize("le millième millésime", lang="fr-fr"),
"1000e millésime")
self.assertEqual(normalize("le trentième anniversaire", lang="fr-fr"),
"30e anniversaire")
def test_spaces(self):
self.assertEqual(normalize(" this is a test"), "this is test")
self.assertEqual(normalize(" this is a test "), "this is test")
self.assertEqual(normalize(" this is one test"),
"this is 1 test")
