input, output = sys.argv[1:3]
# build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens)
# takes about 8h on a macbook pro
wiki = gensim.corpora.WikiCorpus('/Users/kofola/gensim/results/enwiki-20100622-pages-articles.xml.bz2',
keep_words = 200000)
# save dictionary and bag-of-words
# another ~8h
wiki.saveAsText(output)
del wiki
# initialize corpus reader and word->id mapping
from gensim.corpora import MmCorpus
id2token = WikiCorpus.loadDictionary(output + '_wordids.txt')
mm = MmCorpus(output + '_bow.mm')
# build tfidf
# ~20min
from gensim.models import TfidfModel
tfidf = TfidfModel(mm, id2word = id2token, normalize = True)
# save tfidf vectors in matrix market format
# ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB
MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt = 10000)
logging.info("finished running %s" % program)
# running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.
id2word = dmlcorpus.DmlCorpus.loadDictionary(config.resultFile('wordids.txt'))
logging.info("loaded %i word ids" % len(id2word))
corpus = MmCorpus(config.resultFile('bow.mm'))
if method == 'tfidf':
model = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True)
model.save(config.resultFile('model_tfidf.pkl'))
elif method == 'lda':
model = ldamodel.LdaModel(corpus, id2word = id2word, numTopics = DIM_LDA)
model.save(config.resultFile('model_lda.pkl'))
elif method == 'lsi':
# first, transform word counts to tf-idf weights
tfidf = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True)
# then find the transformation from tf-idf to latent space
model = lsimodel.LsiModel(tfidf[corpus], id2word = id2word, numTopics = DIM_LSI)
model.save(config.resultFile('model_lsi.pkl'))
elif method == 'rp':
# first, transform word counts to tf-idf weights
tfidf = tfidfmodel.TfidfModel(corpus, id2word = id2word, normalize = True)
# then find the transformation from tf-idf to latent space
model = rpmodel.RpModel(tfidf[corpus], id2word = id2word, numTopics = DIM_RP)
model.save(config.resultFile('model_rp.pkl'))
else:
raise ValueError('unknown topic extraction method: %s' % repr(method))
MmCorpus.saveCorpus(config.resultFile('corpus_%s.mm' % method), model[corpus])
logging.info("finished running %s" % program)
print globals()['__doc__'] % locals()
sys.exit(1)
input, output = sys.argv[1:3]
# build dictionary. only keep 200k most frequent words (out of total ~7m unique tokens)
# takes about 8h on a macbook pro
wiki = WikiCorpus(input, keep_words=200000)
# save dictionary and bag-of-words
# another ~8h
wiki.saveAsText(output)
del wiki
# initialize corpus reader and word->id mapping
from gensim.corpora import MmCorpus
id2token = WikiCorpus.loadDictionary(output + '_wordids.txt')
mm = MmCorpus(output + '_bow.mm')
# build tfidf
# ~20min
from gensim.models import TfidfModel
tfidf = TfidfModel(mm, id2word=id2token, normalize=True)
# save tfidf vectors in matrix market format
# ~1.5h; result file is 14GB! bzip2'ed down to 4.5GB
MmCorpus.saveCorpus(output + '_tfidf.mm', tfidf[mm], progressCnt=10000)
logging.info("finished running %s" % program)
# running lsi (chunks=20000, numTopics=400) on wiki_tfidf then takes about 14h.
