def trial(train,n,p,probs): """n is n-gram size. p is whether it's log or arbitrary precision""" ngram.set_fractions(p) sentence=ngram.make_sentence(probs[0:(n+1)]) pp=ngram.perplexity(probs[0:(n+1)],sentence) time='NA' return [n,p,pp,time,' '.join(sentence)]
def run(train,nmax,reps,out): #Get probabilities with arbitrary precision fh = open(train) ngram.set_fractions(True) probs_ap=ngram.probabilities(ngram.good_turing(ngram.ngram(nmax,filters.unk(filters.shakespeare(fh))))) #Get probabilities with logs fh = open(train) ngram.set_fractions(False) probs_log=ngram.probabilities(ngram.good_turing(ngram.ngram(nmax,filters.unk(filters.shakespeare(fh))))) #Make sentences sentence_generation(train,out,nmax,reps,probs_ap,probs_log)