__author__ = 'qrr' #generate random sentence using simple N-gram #do experiments and compare using different N and different corpus import corpus.lyric_corpus.corpus_access as corpus from nGram.nGramModel import NgramModel # corpus, trainCorpus, testCorpus, devCorpus = corpus.loadCorpus() # corpus, trainCorpus, testCorpus, devCorpus = corpus.loadCorpus("POP") corpus, trainCorpus, testCorpus, devCorpus = corpus.loadCorpus("ROCK") lm = NgramModel(4, corpus) vocabularyNum = len(set(corpus)) print ("vocabulary number:", vocabularyNum) print ("4-gram") for i in range(15): print (lm.generateRandomSentence())
__author__ = 'Salma' import corpus.lyric_corpus.corpus_access as corpus from nGram.NgramTagModel import NgramTagModel from nGram.nGramModel import NgramModel from nltk.probability import * import random import nltk.corpus #song corpus corpus, trainCorpus, testCorpus, testSents = corpus.loadCorpus() #brow corpus for comaprison brown = nltk.corpus.brown.words() size = int(len(brown) * 0.8) trainBrown = brown[:size] testBrown = brown[size:] #Perplexity for different N-grams with back-off for i in range(1,5): if i==1: lm1 = NgramModel( i,trainCorpus, LaplaceProbDist, False, False) lm2 = NgramModel(i, trainCorpus, LaplaceProbDist, False, False) else : lm1 = NgramModel( i,trainCorpus, MLEProbDist, False, True) lm2 = NgramModel(i, trainCorpus, MLEProbDist, False, True) print("N-gram :", i) print("Brown perplexity: ", lm1.perplexity(testBrown)) print("Song corpus: ", lm2.perplexity(testCorpus))
__author__ = 'Salma' #from nGram import NgramTagModel,nGramModel import corpus.lyric_corpus.corpus_access as corpus from nGram.NgramTagModel import NgramTagModel trainCorpus, testCorpus, devCorpus = corpus.loadCorpus("POP") # trainCorpus, testCorpus, devCorpus = corpus.loadCorpus("ROCK") # print (testCorpus) tm = NgramTagModel(3,2,trainCorpus) testTag = tm.tagTestCorpus(testCorpus) context = tm.getRandomContext(testTag) print("Context", context) print(tm.nextWord(context)) # print(sents)