class SpellChecker:
def __init__(self,
language='en_US',
hunspell_data_dir='./hunspell',
n_jobs=1):
SpellChecker.get_dict(language, hunspell_data_dir)
self.hunspell = Hunspell(language,
hunspell_data_dir=hunspell_data_dir,
disk_cache_dir=os.path.join(
hunspell_data_dir, 'cache'))
self.hunspell.set_concurrency(n_jobs)
self.substitutes = dict()
def spell_check(self, tokenized_corpus_2d):
tokens = {t for iterable in tokenized_corpus_2d for t in iterable}
new_tokens = tokens - self.substitutes.keys()
correct_tokens = {t for t in new_tokens if self.hunspell.spell(t)}
self.substitutes.update(map(lambda t: (t, t), correct_tokens))
tokens_to_check = new_tokens - correct_tokens
suggestions = self.hunspell.bulk_suggest(tokens_to_check)
self.substitutes.update(
map(lambda kv: (kv[0], kv[0]) if not kv[1] else (kv[0], kv[1][0]),
suggestions.items()))
new_corpus = [[self.substitutes[token] for token in iterable]
for iterable in tokenized_corpus_2d]
return new_corpus
@staticmethod
def get_dict(language, data_dir):
os.makedirs(data_dir, exist_ok=True)
for ext in ['aff', 'dic']:
path = os.path.join(data_dir, '%s.%s' % (language, ext))
if not os.path.exists(path):
r = get(
'https://raw.githubusercontent.com/LibreOffice/dictionaries/master/%s/%s.%s'
% (language, language, ext))
if r.status_code == 404:
l = language[0:language.find('_')]
r = get(
'https://raw.githubusercontent.com/LibreOffice/dictionaries/master/%s/%s.%s'
% (l, language, ext))
r.raise_for_status()
f = open(path, 'wb')
f.write(r.content)
f.close()
def __del__(self):
self.hunspell.save_cache() # For future program executions.
class HunspellTest(unittest.TestCase):
def assertRegexpSearch(self, *args, **kwargs):
if PY3:
self.assertRegex(*args, **kwargs)
else:
self.assertRegexpMatches(*args, **kwargs)
def setUp(self):
self.h = Hunspell('test', hunspell_data_dir=DICT_DIR)
def tearDown(self):
try:
del self.h
except AttributeError:
pass
def assertAllIn(self, checked, expected):
self.assertTrue(all(x in expected for x in checked),
u"{} not all found in {}".format(checked, expected))
def test_create_destroy(self):
del self.h
def test_missing_dict(self):
with self.assertRaises(HunspellFilePathError):
Hunspell('not_avail', hunspell_data_dir=DICT_DIR)
@patch('os.path.isfile', return_value=True)
@patch('os.access', return_value=True)
def test_bad_path_encoding(self, *mocks):
if PY3:
with self.assertRaises(HunspellFilePathError):
Hunspell('not_checked',
hunspell_data_dir=u'bad/\udcc3/decoding')
else:
# Python 2 just make an illegal string instead of raising
with captured_c_stderr_file() as caperr:
Hunspell('not_checked',
hunspell_data_dir=u'bad/\udcc3/decoding')
with open(caperr, 'r') as err:
self.assertRegexpSearch(err.read(), r'error:[^\n]*bad/[^\n]*/decoding')
@patch('hunspell.hunspell.WIN32_LONG_PATH_PREFIX', '/not/valid')
def test_windows_utf_8_encoding_applies_prefix(self, *mocks):
with captured_c_stderr_file() as caperr:
with patch("os.name", 'nt'):
# If python file existance checks used prefix, this would raise a HunspellFilePathError
Hunspell('test', system_encoding='UTF-8')
with open(caperr, 'r') as err:
# But the Hunspell library lookup had the prefix applied
self.assertRegexpSearch(err.read(), r'error:[^\n]*/not/valid[^\n]*')
def test_spell(self):
self.assertFalse(self.h.spell('dpg'))
self.assertTrue(self.h.spell('dog'))
def test_spell_utf8(self):
self.assertTrue(self.h.spell(u'café'))
self.assertFalse(self.h.spell(u'uncafé'))
def test_spell_empty(self):
self.assertTrue(self.h.spell(''))
def test_suggest(self):
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
suggest = self.h.suggest('dpg')
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_utf8(self):
required = (u'café', u'Cerf')
for variant in ('cefé', u'cefé'):
suggest = self.h.suggest(variant)
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_empty(self):
self.assertEqual(self.h.suggest(''), ())
def test_stem(self):
self.assertEqual(self.h.stem('dog'), ('dog',))
self.assertEqual(self.h.stem('permanently'), ('permanent',))
def test_add(self):
word = 'outofvocabularyword'
self.assertEqual(self.h.spell(word), False)
self.h.add(word)
self.assertEqual(self.h.spell(word), True)
typo = word + 'd'
self.assertAllIn([word], self.h.suggest(typo))
def test_bulk_suggest(self):
self.h.set_concurrency(3)
suggest = self.h.bulk_suggest(['dog', 'dpg'])
self.assertEqual(sorted(suggest.keys()), ['dog', 'dpg'])
self.assertIsInstance(suggest['dog'], tuple)
self.assertAllIn(('dog',), suggest['dog'])
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
self.assertIsInstance(suggest['dpg'], tuple)
self.assertAllIn(required, suggest['dpg'])
checked = ['bjn', 'dog', 'dpg', 'dyg', 'foo', 'frg', 'opg', 'pgg', 'qre', 'twg']
suggest = self.h.bulk_suggest(checked)
self.assertEqual(sorted(suggest.keys()), checked)
def test_bulk_stem(self):
self.h.set_concurrency(3)
self.assertDictEqual(self.h.bulk_stem(['dog', 'permanently']), {
'permanently': ('permanent',),
'dog': ('dog',)
})
self.assertDictEqual(self.h.bulk_stem(['dog', 'twigs', 'permanently', 'unrecorded']), {
'unrecorded': ('recorded',),
'permanently': ('permanent',),
'twigs': ('twig',),
'dog': ('dog',)
})
def test_non_overlapping_caches(self):
test_suggest = self.h.suggest('testing')
test_stem = self.h.stem('testing')
self.h._suggest_cache['made-up'] = test_suggest
self.assertEqual(self.h.suggest('made-up'), test_suggest)
self.h._stem_cache['made-up'] = test_stem
self.assertEqual(self.h.stem('made-up'), test_stem)
h2 = Hunspell('en_US', hunspell_data_dir=DICT_DIR)
self.assertNotEqual(h2.suggest('made-up'), test_suggest)
self.assertNotEqual(h2.stem('made-up'), test_stem)
def test_overlapping_caches(self):
test_suggest = self.h.suggest('testing')
test_stem = self.h.stem('testing')
self.h._suggest_cache['made-up'] = test_suggest
self.assertEqual(self.h.suggest('made-up'), test_suggest)
self.h._stem_cache['made-up'] = test_stem
self.assertEqual(self.h.stem('made-up'), test_stem)
del self.h
self.h = Hunspell('test', hunspell_data_dir=DICT_DIR)
self.assertEqual(self.h.suggest('made-up'), test_suggest)
self.assertEqual(self.h.stem('made-up'), test_stem)
def test_save_caches_persistance(self):
temp_dir = tempfile.mkdtemp()
try:
h1 = Hunspell('test',
hunspell_data_dir=DICT_DIR,
disk_cache_dir=temp_dir,
cache_manager='disk_hun')
test_suggest = h1.suggest('testing')
test_stem = h1.stem('testing')
h1._suggest_cache['made-up'] = test_suggest
self.assertEqual(h1.suggest('made-up'), test_suggest)
h1._stem_cache['made-up'] = test_stem
self.assertEqual(h1.stem('made-up'), test_stem)
h1.save_cache()
del h1
cacheman = get_cache_manager('disk_hun')
cacheman.deregister_all_caches()
self.assertEqual(len(cacheman.cache_by_name), 0)
h2 = Hunspell('test',
hunspell_data_dir=DICT_DIR,
disk_cache_dir=temp_dir,
cache_manager='disk_hun')
self.assertNotEqual(len(h2._suggest_cache), 0)
self.assertNotEqual(len(h2._stem_cache), 0)
self.assertEqual(h2.suggest('made-up'), test_suggest)
self.assertEqual(h2.stem('made-up'), test_stem)
finally:
shutil.rmtree(temp_dir) # Nuke temp content
def test_clear_caches_persistance(self):
temp_dir = tempfile.mkdtemp()
try:
h1 = Hunspell('test',
hunspell_data_dir=DICT_DIR,
disk_cache_dir=temp_dir,
cache_manager='disk_hun')
test_suggest = h1.suggest('testing')
test_stem = h1.stem('testing')
h1._suggest_cache['made-up'] = test_suggest
self.assertEqual(h1.suggest('made-up'), test_suggest)
h1._stem_cache['made-up'] = test_stem
self.assertEqual(h1.stem('made-up'), test_stem)
h1.save_cache()
h1.clear_cache()
del h1
cacheman = get_cache_manager('disk_hun')
cacheman.deregister_all_caches()
self.assertEqual(len(cacheman.cache_by_name), 0)
h2 = Hunspell('test',
hunspell_data_dir=DICT_DIR,
disk_cache_dir=temp_dir,
cache_manager='disk_hun')
self.assertEqual(len(h2._suggest_cache), 0)
self.assertEqual(len(h2._stem_cache), 0)
self.assertNotEqual(h2.suggest('made-up'), test_suggest)
self.assertNotEqual(h2.stem('made-up'), test_stem)
finally:
shutil.rmtree(temp_dir) # Nuke temp content
def test_clear_caches_non_peristance(self):
test_suggest = self.h.suggest('testing')
test_stem = self.h.stem('testing')
self.h._suggest_cache['made-up'] = test_suggest
self.assertEqual(self.h.suggest('made-up'), test_suggest)
self.h._stem_cache['made-up'] = test_stem
self.assertEqual(self.h.stem('made-up'), test_stem)
self.h.clear_cache()
del self.h
self.h = Hunspell('test', hunspell_data_dir=DICT_DIR)
self.assertNotEqual(self.h.suggest('made-up'), test_suggest)
self.assertNotEqual(self.h.stem('made-up'), test_stem)
class HunspellTest(unittest.TestCase):
def setUp(self):
self.h = Hunspell('test', hunspell_data_dir=DICT_DIR)
def tearDown(self):
try:
del self.h
except AttributeError:
pass
def assertAllIn(self, checked, expected):
self.assertTrue(all(x in expected for x in checked),
u"{} not all found in {}".format(checked, expected))
def test_create_destroy(self):
del self.h
def test_missing_dict(self):
with self.assertRaises(IOError):
Hunspell('not_avail', hunspell_data_dir=DICT_DIR)
def test_spell(self):
self.assertFalse(self.h.spell('dpg'))
self.assertTrue(self.h.spell('dog'))
def test_spell_utf8(self):
self.assertTrue(self.h.spell(u'café'))
self.assertFalse(self.h.spell(u'uncafé'))
def test_spell_empty(self):
self.assertTrue(self.h.spell(''))
def test_suggest(self):
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
suggest = self.h.suggest('dpg')
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_utf8(self):
required = (u'café', u'Cerf')
for variant in ('cefé', u'cefé'):
suggest = self.h.suggest(variant)
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_empty(self):
self.assertEqual(self.h.suggest(''), ())
def test_stem(self):
self.assertEqual(self.h.stem('dog'), ('dog',))
self.assertEqual(self.h.stem('permanently'), ('permanent',))
def test_bulk_suggest(self):
self.h.set_concurrency(3)
suggest = self.h.bulk_suggest(['dog', 'dpg'])
self.assertEqual(sorted(suggest.keys()), ['dog', 'dpg'])
self.assertIsInstance(suggest['dog'], tuple)
self.assertAllIn(('dog',), suggest['dog'])
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
self.assertIsInstance(suggest['dpg'], tuple)
self.assertAllIn(required, suggest['dpg'])
checked = ['bjn', 'dog', 'dpg', 'dyg', 'foo', 'frg', 'opg', 'pgg', 'qre', 'twg']
suggest = self.h.bulk_suggest(checked)
self.assertEqual(sorted(suggest.keys()), checked)
def test_bulk_stem(self):
self.h.set_concurrency(3)
self.assertDictEqual(self.h.bulk_stem(['dog', 'permanently']), {
'permanently': ('permanent',),
'dog': ('dog',)
})
self.assertDictEqual(self.h.bulk_stem(['dog', 'twigs', 'permanently', 'unrecorded']), {
'unrecorded': ('recorded',),
'permanently': ('permanent',),
'twigs': ('twig',),
'dog': ('dog',)
})
class HunspellTest(unittest.TestCase):
def setUp(self):
self.h = Hunspell('test', hunspell_data_dir=DICT_DIR)
def tearDown(self):
try:
del self.h
except AttributeError:
pass
def assertAllIn(self, checked, expected):
self.assertTrue(all(x in expected for x in checked),
u"{} not all found in {}".format(checked, expected))
def test_create_destroy(self):
del self.h
def test_missing_dict(self):
with self.assertRaises(IOError):
Hunspell('not_avail', hunspell_data_dir=DICT_DIR)
def test_spell(self):
self.assertFalse(self.h.spell('dpg'))
self.assertTrue(self.h.spell('dog'))
def test_spell_utf8(self):
self.assertTrue(self.h.spell(u'café'))
self.assertFalse(self.h.spell(u'uncafé'))
def test_spell_empty(self):
self.assertTrue(self.h.spell(''))
def test_suggest(self):
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
suggest = self.h.suggest('dpg')
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_utf8(self):
required = (u'café', u'Cerf')
for variant in ('cefé', u'cefé'):
suggest = self.h.suggest(variant)
self.assertIsInstance(suggest, tuple)
self.assertAllIn(required, suggest)
def test_suggest_empty(self):
self.assertEqual(self.h.suggest(''), ())
def test_stem(self):
self.assertEqual(self.h.stem('dog'), ('dog', ))
self.assertEqual(self.h.stem('permanently'), ('permanent', ))
def test_bulk_suggest(self):
self.h.set_concurrency(3)
suggest = self.h.bulk_suggest(['dog', 'dpg'])
self.assertEqual(sorted(suggest.keys()), ['dog', 'dpg'])
self.assertIsInstance(suggest['dog'], tuple)
self.assertAllIn(('dog', ), suggest['dog'])
required = ('dog', 'pg', 'deg', 'dig', 'dpt', 'dug', 'mpg', 'd pg')
self.assertIsInstance(suggest['dpg'], tuple)
self.assertAllIn(required, suggest['dpg'])
checked = [
'bjn', 'dog', 'dpg', 'dyg', 'foo', 'frg', 'opg', 'pgg', 'qre',
'twg'
]
suggest = self.h.bulk_suggest(checked)
self.assertEqual(sorted(suggest.keys()), checked)
def test_bulk_stem(self):
self.h.set_concurrency(3)
self.assertDictEqual(self.h.bulk_stem(['dog', 'permanently']), {
'permanently': ('permanent', ),
'dog': ('dog', )
})
self.assertDictEqual(
self.h.bulk_stem(['dog', 'twigs', 'permanently', 'unrecorded']), {
'unrecorded': ('recorded', ),
'permanently': ('permanent', ),
'twigs': ('twig', ),
'dog': ('dog', )
})
class CyHunspell():
'''
Спеллер на основе cython версии hunspell
>>> word_en = 'cookbok'
>>> word_ru = 'поваринная'
>>> speller_en = CyHunspell(lang="en")
>>> speller_en.spell(word_en)
False
>>> speller_en.suggest(word_en)
('cookbook', 'copybook', 'codebook', 'Cook', 'cook')
>>> speller_en.replace(word_en)
'cookbook'
>>> speller_ru = CyHunspell(lang="ru")
>>> speller_ru.spell(word_ru)
False
>>> speller_ru.suggest(word_ru)
('поваренная',)
>>> speller_ru.replace(word_ru)
'поваренная'
'''
langs = {'ru': 'ru_RU', 'en': 'en_US'}
def __init__(
self,
lang='en',
max_dist=2,
cpu=os.cpu_count(),
# cache_manager="hunspell",disk_cache_dir=None,
# hunspell_data_dir=None,system_encoding=None
spell_kwargs={}):
self.lang = self.langs.get(lang, lang)
self.spell_dict = Hunspell(self.lang, **spell_kwargs)
self.max_dist = max_dist
self.spell_dict.set_concurrency(cpu)
def spell(self, word):
try:
result = self.spell_dict.spell(word)
except UnicodeEncodeError as err:
result = None
return result
def suggest(self, word):
try:
result = self.spell_dict.suggest(word)
except UnicodeEncodeError as err:
result = tuple()
return result
def replace(self, word, max_dist=None):
max_dist = max_dist if max_dist is not None else self.max_dist
if self.spell(word):
return word
suggestions = self.suggest(word)
if (suggestions and edit_distance(word, suggestions[0]) <= max_dist):
return suggestions[0]
else:
return word
