def make_dims(head=3630013, series_len=19):
my_lsi_v = entropy.lsi_v("../data/warehouse/gensim_complete_corpus.lsi.%s.projection.v." % (head, ), series_len)
fh = open("../data/warehouse/gensim_complete_corpus.lsi.%s.projection.vt2.pickle" % (head, ), "w")
for i, d in enumerate(my_lsi_v._iter_dims()):
if i%10==0:
print " dim", i
cPickle.dump(d, fh)
fh.close()
def make_dim_chunks(head=3630013, series_len=19, start=0, stop=50, restrict_disease=True):
my_lsi_v = entropy.lsi_v("../data/warehouse/gensim_%s_corpus.lsi.%s.projection.vgood." % (["complete", "disease"][restrict_disease], head, ), series_len)
fh = open("../data/warehouse/gensim_%s_corpus.lsi.%s.projection.vgoodt2.%03d-%03d.pickle" % (["complete", "disease"][restrict_disease], head, start, stop, ), "w")
for i, d in enumerate(my_lsi_v._dim_all_docs(start, stop)):
if i%10==0:
print " dim", i
cPickle.dump(list(d), fh)
fh.close()