def dump_thesaurus(theme='source', topn=1000): db = dbcon.connect_torndb() seg = Segmenter() stopwords = stopword.get_standard_stopwords() tags = set(x.name for x in db.query('select name from tag where type>11001;')) vocab = {} if theme == 'source': query = 'select * from source_company where (active is null or active="Y");' else: query = 'select * from source_company where (active is null or active="Y");' for index, item in enumerate(db.iter(query)): for word in set(filter(lambda x: x not in stopwords and len(x) > 1 and not x.isnumeric() and x not in tags and x.strip(), seg.cut(item.description))): vocab[word] = vocab.get(word, 0) + 1 if index % 10000 == 0: low = [x[0] for x in vocab.iteritems() if x[1] < 20] for lowword in low: vocab.pop(lowword) print index, 'processed, size of vocab', len(vocab) db.close() vocab = sorted(vocab.iteritems(), key=lambda x: x[1], reverse=True)[:topn] with codecs.open(os.path.join(os.path.split(os.path.realpath(__file__))[0], 'thesaurus/%s.%s.lowidf' % (theme, topn)), 'w', 'utf-8') as fo: fo.write('\n'.join([x[0] for x in vocab]))
def __init__(self): self.db = dbcon.connect_torndb() self.tagger = Tagger() self.stopwords = stopword.get_standard_stopwords() self.source = json.load(codecs.open(os.path.join(os.path.split(os.path.realpath(__file__))[0], 'dumps/news.domain'), encoding='utf-8')) self.failed = 0
class UniversalIndexCreator(object): stopwords = stopword.get_standard_stopwords() seg = Segmenter() nameseg = NameSegmenter() def __init__(self, es=None): global logger_universal_index if not es: host, port = tsbconfig.get_es_config() self.es = Elasticsearch([{'host': host, 'port': port}]) else: self.es = es self.topic_tags = {} logger_universal_index.info('Universal Index Creator inited') def __check(self): global logger_universal_index if not self.es.indices.exists(["xiniudata2"]): logger_universal_index.info('Creating index xiniudata2') self.es.indices.create("xiniudata2") logger_universal_index.info('Created') self.es.indices.put_mapping("universal", mappings.get_universal_company_mapping(), "xiniudata2") logger_universal_index.info('Universal Company mapping created') def create_indice(self): global logger_universal_index self.__check() db = dbcon.connect_torndb() self.topic_tags = dbutil.get_topic_corresponding_tags(db) logger_universal_index.info('Start to create indice') logger_universal_index.info(str(self.es.info())) logger_universal_index.info('ES Config %s' % str(tsbconfig.get_es_config())) for cid in dbutil.get_all_company_id(db): try: self.create_single(db, cid) logger_universal_index.info( '%s index created, %s' % (cid, dbutil.get_company_name(db, cid))) except Exception, e: logger_universal_index.exception('%s failed # %s' % (cid, e)) db.close()
def __init__(self, opt=None): if not isinstance(opt, dict): opt = {} if opt.get('segmenter'): self.seg = opt.get('segmenter') else: self.seg = Segmenter() self.vectorizer = TfidfVectorizer( sublinear_tf=True, stop_words=stopword.get_standard_stopwords(), max_df=opt.get('max_df', 0.5), min_df=opt.get('min_df', 50), max_features=5000) self.selector = SelectKBest(chi2, k=opt.get('topk', 'all'))
def __init__(self): self.stopwords = stopword.get_standard_stopwords() self.stopwords.update(punctuation)
import codecs import time import re from random import random from common import nlpconfig from common.zhtools import stopword from common.zhtools import hants from common.zhtools.postagger import Tagger tagger = Tagger('ltp') doc_len_threshold = 10 year = re.compile(u'\d+年') month = re.compile(u'\d+月') day = re.compile(u'\d+日') stopwords = stopword.get_standard_stopwords() class Corpus(object): def __init__(self, extrc_func=None, dirs=None): self.extract = extrc_func if extrc_func else lambda x: x.strip() self.dirs = dirs def __iter__(self): for subdir in self.dirs: for f in os.listdir(subdir): if u'DS_Store' in f: continue