Beispiel #1
0
def trial(train,n,p,probs):
	"""n is n-gram size. p is whether it's log or arbitrary precision"""
	ngram.set_fractions(p)
	sentence=ngram.make_sentence(probs[0:(n+1)])
	pp=ngram.perplexity(probs[0:(n+1)],sentence)
	time='NA'
	return [n,p,pp,time,' '.join(sentence)]
Beispiel #2
0
def run(train,nmax,reps,out):
	#Get probabilities with arbitrary precision
	fh = open(train)
	ngram.set_fractions(True)
	probs_ap=ngram.probabilities(ngram.good_turing(ngram.ngram(nmax,filters.unk(filters.shakespeare(fh)))))
	
	#Get probabilities with logs
	fh = open(train)
	ngram.set_fractions(False)
	probs_log=ngram.probabilities(ngram.good_turing(ngram.ngram(nmax,filters.unk(filters.shakespeare(fh)))))
	
	#Make sentences
	sentence_generation(train,out,nmax,reps,probs_ap,probs_log)