def make_weird_model(text_filename): print "loading %s into memcached" % text_filename import bigrams # model = MemcacheLM(dont_load_info=True) # model = TTLM() model = RedisLM() bigrams.collect_statistics_into_model(fileinput.input(text_filename), model) return model
def make_weird_model(text_filename): print "loading %s into memcached" % text_filename import bigrams #model = MemcacheLM(dont_load_info=True) #model = TTLM() model = RedisLM() bigrams.collect_statistics_into_model(fileinput.input(text_filename), model) return model
def make_tokyo_model(text_filename): import bigrams model = TokyoLM(readonly=False) #model = BDBLM() #model = LocalLM() iter = fileinput.input(text_filename) #iter = occasional(iter, lambda: model.sync()) bigrams.collect_statistics_into_model(iter, model) model.sync() return model