def _get_epub_standard_word_count(iterator, lang='en'):
'''
This algorithm counts individual words instead of pages
'''
book_text = _read_epub_contents(iterator, strip_html=True)
try:
from calibre.spell.break_iterator import count_words
wordcount = count_words(book_text, lang)
logger.debug('\tWord count - count_words method:%s' % wordcount)
except:
try: # The above method is new and no-one will have it as of 08/01/2016. Use an older method for a beta.
from calibre.spell.break_iterator import split_into_words_and_positions
wordcount = len(split_into_words_and_positions(book_text, lang))
logger.debug(
'\tWord count - split_into_words_and_positions method:%s' %
wordcount)
except:
from calibre.utils.wordcount import get_wordcount_obj
wordcount = get_wordcount_obj(book_text)
wordcount = wordcount.words
logger.debug('\tWord count - old method:%s' % wordcount)
return wordcount
def test_break_iterator(self):
' Test the break iterator '
from calibre.spell.break_iterator import split_into_words as split, index_of, split_into_words_and_positions, count_words
for q in (
'one two three',
' one two three',
'one\ntwo three ',
):
self.ae(split(str(q)), ['one', 'two', 'three'],
'Failed to split: %r' % q)
self.ae(split('I I\'m'), ['I', "I'm"])
self.ae(split('out-of-the-box'), ['out-of-the-box'])
self.ae(split('-one two-'), ['-one', 'two-'])
self.ae(split('-one a-b-c-d e'), ['-one', 'a-b-c-d', 'e'])
self.ae(split('-one -a-b-c-d- e'), ['-one', '-a-b-c-d-', 'e'])
self.ae(split_into_words_and_positions('one \U0001f431 three'),
[(0, 3), (6, 5)])
self.ae(count_words('a b c d e f'), 6)
for needle, haystack, pos in (
('word', 'a word b', 2),
('word', 'a word', 2),
('one-two', 'a one-two punch', 2),
('one-two', 'one-two punch', 0),
('one-two', 'one-two', 0),
('one', 'one-two one', 8),
('one-two', 'one-two-three one-two', 14),
('one', 'onet one', 5),
('two', 'one-two two', 8),
('two', 'two-one two', 8),
('-two', 'one-two -two', 8),
('-two', 'two', -1),
('i', 'i', 0),
('i', 'six i', 4),
('i', '', -1),
('', '', -1),
('', 'i', -1),
('i', 'six clicks', -1),
('i', '\U0001f431 i', 2),
('-a', 'b -a', 2),
('a-', 'a-b a- d', 4),
('-a-', 'b -a -a-', 5),
('-a-', '-a-', 0),
('-a-', 'a-', -1),
('-a-', '-a', -1),
('-a-', 'a', -1),
('a-', 'a-', 0),
('-a', '-a', 0),
('a-b-c-', 'a-b-c-d', -1),
('a-b-c-', 'a-b-c-.', 0),
('a-b-c-', 'a-b-c-d a-b-c- d', 8),
):
fpos = index_of(needle, haystack)
self.ae(
pos, fpos, 'Failed to find index of %r in %r (%d != %d)' %
(needle, haystack, pos, fpos))
def _get_epub_standard_word_count(iterator, lang='en'):
'''
This algorithm counts individual words instead of pages
'''
book_text = _read_epub_contents(iterator, strip_html=True)
try:
from calibre.spell.break_iterator import count_words
wordcount = count_words(book_text, lang)
logger.debug('\tWord count - count_words method:%s'%wordcount)
except:
try: # The above method is new and no-one will have it as of 08/01/2016. Use an older method for a beta.
from calibre.spell.break_iterator import split_into_words_and_positions
wordcount = len(split_into_words_and_positions(book_text, lang))
logger.debug('\tWord count - split_into_words_and_positions method:%s'%wordcount)
except:
from calibre.utils.wordcount import get_wordcount_obj
wordcount = get_wordcount_obj(book_text)
wordcount = wordcount.words
logger.debug('\tWord count - old method:%s'%wordcount)
return wordcount