def try_justext(tree, url, target_language):
'''Second safety net: try with the generic algorithm justext'''
result_body = etree.Element('body')
# determine language
if target_language is not None and target_language in JUSTEXT_LANGUAGES:
justext_stoplist = get_stoplist(JUSTEXT_LANGUAGES[target_language])
else:
justext_stoplist = JT_STOPLIST
# extract
try:
paragraphs = custom_justext(tree, justext_stoplist)
except ValueError as err: # not an XML element: HtmlComment
LOGGER.error('justext %s %s', err, url)
result_body = None
else:
for paragraph in [p for p in paragraphs if not p.is_boilerplate]:
#if duplicate_test(paragraph) is not True:
elem, elem.text = etree.Element('p'), paragraph.text
result_body.append(elem)
return result_body
def test_get_real_stoplist(self):
stopwords = get_stoplist("Slovak")
tools.assert_true(len(stopwords) > 0)
def jt_stoplist_init():
'Retrieve and return the content of all JusText stoplists'
stoplist = set()
for language in get_stoplists():
stoplist.update(get_stoplist(language))
return stoplist
def test_get_missing_stoplist(self):
with pytest.raises(ValueError):
get_stoplist("Klingon")
def test_get_real_stoplist(self):
stopwords = get_stoplist("Slovak")
assert len(stopwords) > 0
def test_get_missing_stoplist(self):
with pytest.raises(ValueError):
get_stoplist("Klingon")
def test_get_real_stoplist(self):
stopwords = get_stoplist("Slovak")
assert len(stopwords) > 0
def test_get_real_stoplist(self):
stopwords = get_stoplist("Slovak")
tools.assert_true(len(stopwords) > 0)