input, output = sys.argv[1:3] # build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens) # takes about 8h on a macbook pro wiki = gensim.corpora.WikiCorpus('/Users/kofola/gensim/results/enwiki-20100622-pages-articles.xml.bz2', keep_words = 200000) # save dictionary and bag-of-words # another ~8h wiki.saveAsText(output) del wiki # initialize corpus reader and word->id mapping from gensim.corpora import MmCorpus id2token = WikiCorpus.loadDictionary(output + '_wordids.txt') mm = MmCorpus(output + '_bow.mm') # build tfidf # ~20min from gensim.models import TfidfModel tfidf = TfidfModel(mm, id2word = id2token, normalize = True) # save tfidf vectors in matrix market format # ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt = 10000) logging.info("finished running %s" % program) # running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.
id2word = dmlcorpus.DmlCorpus.loadDictionary(config.resultFile('wordids.txt')) logging.info("loaded %i word ids" % len(id2word)) corpus = MmCorpus(config.resultFile('bow.mm')) if method == 'tfidf': model = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True) model.save(config.resultFile('model_tfidf.pkl')) elif method == 'lda': model = ldamodel.LdaModel(corpus, id2word = id2word, numTopics = DIM_LDA) model.save(config.resultFile('model_lda.pkl')) elif method == 'lsi': # first, transform word counts to tf-idf weights tfidf = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True) # then find the transformation from tf-idf to latent space model = lsimodel.LsiModel(tfidf[corpus], id2word = id2word, numTopics = DIM_LSI) model.save(config.resultFile('model_lsi.pkl')) elif method == 'rp': # first, transform word counts to tf-idf weights tfidf = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True) # then find the transformation from tf-idf to latent space model = rpmodel.RpModel(tfidf[corpus], id2word = id2word, numTopics = DIM_RP) model.save(config.resultFile('model_rp.pkl')) else: raise ValueError('unknown topic extraction method: %s' % repr(method)) MmCorpus.saveCorpus(config.resultFile('corpus_%s.mm' % method), model[corpus]) logging.info("finished running %s" % program)
print globals()['__doc__'] % locals() sys.exit(1) input, output = sys.argv[1:3] # build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens) # takes about 8h on a macbook pro wiki = WikiCorpus(input, keep_words=200000) # save dictionary and bag-of-words # another ~8h wiki.saveAsText(output) del wiki # initialize corpus reader and word->id mapping from gensim.corpora import MmCorpus id2token = WikiCorpus.loadDictionary(output + '_wordids.txt') mm = MmCorpus(output + '_bow.mm') # build tfidf # ~20min from gensim.models import TfidfModel tfidf = TfidfModel(mm, id2word=id2token, normalize=True) # save tfidf vectors in matrix market format # ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt=10000) logging.info("finished running %s" % program) # running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.