class MeCabMultiProcessing(object):
pool = None
def set_pool(self):
self.pool = Pool(4)
def count_doc(self, doc, **kwds):
if not self.pool:
self.set_pool()
f = partial(count_word, **kwds)
return reduce(lambda x, y: x + y, self.pool.imap_unordered(f, doc))
class MeCabMultiProcessing(object):
pool = None
def set_pool(self):
self.pool = Pool(4)
def count_doc(self, doc, **kwds):
if not self.pool:
self.set_pool()
f = partial(count_word, **kwds)
return reduce(lambda x, y: x+y, self.pool.imap_unordered(f, doc))
class Ngramer(object):
pool = None
def set_pool(self):
self.pool = Pool(4)
def to_ngrams(self, item, max_n=0, min_n=2):
if max_n < 1:
max_n = len(item)
args = zip_longest([item]*((max_n + 1) - min_n), range(min_n, max_n+1))
if not self.pool:
self.set_pool()
return list(self.pool.imap(_to_ngram_unpack, args))
class Ngramer(object):
pool = None
def set_pool(self):
self.pool = Pool(4)
def to_ngrams(self, item, max_n=0, min_n=2):
if max_n < 1:
max_n = len(item)
args = zip_longest([item] * ((max_n + 1) - min_n),
range(min_n, max_n + 1))
if not self.pool:
self.set_pool()
return list(self.pool.imap(_to_ngram_unpack, args))
def set_pool(self):
self.pool = Pool(4)
def set_pool(self):
self.pool = Pool(4)
