def testRawProbabilities(self): model = NgramsModel(2) model.count( self.corpusfile ) probas = model.probabilities( method="raw" ) self.assertEqual(len(probas), 2) unigram = probas[0] for token,value,bo in unigram: if token=="a": self.assertEqual(value, 15) if token=='b': self.assertEqual(value, 10) if token=='c': self.assertEqual(value, 4) if token=='d': self.assertEqual(value, 3) if token==START_SENT_SYMBOL: self.assertEqual(value, 0) if token==END_SENT_SYMBOL: self.assertEqual(value, 3) bigram = probas[1] for token,value,bo in bigram: if token=="a b": self.assertEqual(value, 7) if token=="b a": self.assertEqual(value, 4) if token==START_SENT_SYMBOL+' a': self.assertEqual(value, 3) if token=='b '+END_SENT_SYMBOL: self.assertEqual(value, 3) probas = model.probabilities( method="lograw" ) self.assertEqual(len(probas), 2) unigram = probas[0] for token,value,bo in unigram: if token=="a": self.assertEqual(value, math.log(15, 10)) if token=='b': self.assertEqual(value, math.log(10, 10)) if token=='c': self.assertEqual(value, math.log(4, 10)) if token=='d': self.assertEqual(value, math.log(3, 10)) if token==START_SENT_SYMBOL: self.assertEqual(value, -99) if token==END_SENT_SYMBOL: self.assertEqual(value, math.log(3, 10)) bigram = probas[1] for token,value,bo in bigram: if token=="a b": self.assertEqual(value, math.log(7, 10)) if token=="b a": self.assertEqual(value, math.log(4, 10)) if token==START_SENT_SYMBOL+' a': self.assertEqual(value, math.log(3, 10)) if token=='b '+END_SENT_SYMBOL: self.assertEqual(value, math.log(3, 10))
def gen_slm_dependencies(self, basename, N=3): """ Generate the dependencies (slm, dictionary) for julius. @param basename (str - IN) the base name of the slm file and of the dictionary file @param N (int) Language model N-gram length. """ dictname = basename + ".dict" slmname = basename + ".arpa" phoneslist = self._phones.split() tokenslist = self._tokens.split() dictpron = DictPron() for token,pron in zip(tokenslist,phoneslist): for variant in pron.split("|"): dictpron.add_pron( token, variant.replace("-"," ") ) if dictpron.is_unk(START_SENT_SYMBOL) is True: dictpron.add_pron( START_SENT_SYMBOL, "sil" ) if dictpron.is_unk(END_SENT_SYMBOL) is True: dictpron.add_pron( END_SENT_SYMBOL, "sil" ) dictpron.save_as_ascii( dictname, False ) # Write the SLM model = NgramsModel(N) model.append_sentences( [self._tokens] ) probas = model.probabilities( method="logml" ) arpaio = ArpaIO() arpaio.set( probas ) arpaio.save( slmname )
def testARPA(self): fn1 = os.path.join(TEMP,"model1.arpa") fn2 = os.path.join(TEMP,"model2.arpa") model = NgramsModel(3) model.count( self.corpusfile ) probas = model.probabilities("logml") arpaio = ArpaIO() arpaio.set( probas ) arpaio.save( fn1 ) slm1 = SLM() slm1.load_from_arpa( fn1 ) slm1.save_as_arpa( fn2 ) slm2 = SLM() slm2.load_from_arpa( fn2 ) m1 = slm1.model m2 = slm2.model self.assertTrue( compare(m1,m2) )
# ---------------------------------------------------------------------------- # Main program # ---------------------------------------------------------------------------- # --------------------------------- # 1. Create a NgramsModel model = NgramsModel( args.n ) if args.r: model.set_vocab( args.r ) # --------------------------------- # 2. Estimate counts of each n-gram model.count( *(args.i) ) # --------------------------------- # 3. Estimate probabilities probas = model.probabilities( args.m ) # --------------------------------- # 4. Write in an ARPA file arpaio = ArpaIO() arpaio.set( probas ) arpaio.save( args.o ) # ---------------------------------------------------------------------------
def testMaximumLikelihoodProbabilities(self): model = NgramsModel(3) model.count( self.corpusfile ) probas = model.probabilities( method="ml" ) self.assertEqual(len(probas), 3) unigram = probas[0] for token,value,bo in unigram: if token=="a": self.assertEqual(round(value,6), 0.428571) if token=="b": self.assertEqual(round(value,6), 0.285714) if token=="c": self.assertEqual(round(value,6), 0.114286) if token=="d": self.assertEqual(round(value,6), 0.085714) if token==START_SENT_SYMBOL: self.assertEqual(round(value,6), 0.) if token==END_SENT_SYMBOL: self.assertEqual(round(value,6), 0.085714) bigram = probas[1] for token,value,bo in bigram: if token=="a b": self.assertEqual(round(value,6), 0.466667) if token=="b a": self.assertEqual(round(value,6), 0.400000) trigram = probas[2] for token,value,bo in trigram: if token=="a b a": self.assertEqual(round(value,6), 0.142857) if token==START_SENT_SYMBOL+"a a": self.assertEqual(round(value,6), 0.500000) if token=="a b"+END_SENT_SYMBOL: self.assertEqual(round(value,6), 0.428571) probas = model.probabilities( method="logml" ) self.assertEqual(len(probas), 3) unigram = probas[0] for token,value,bo in unigram: if token=="a": self.assertEqual(round(value,6), round(math.log(0.42857143,10),6)) if token=="b": self.assertEqual(round(value,6), round(math.log(0.28571429,10),6)) if token=="c": self.assertEqual(round(value,6), round(math.log(0.11428571,10),6)) if token=="d": self.assertEqual(round(value,6), round(math.log(0.08571429,10),6)) if token==START_SENT_SYMBOL: self.assertEqual(round(value,6), -99.000000) if token==END_SENT_SYMBOL: self.assertEqual(round(value,6), round(math.log(0.08571429,10),6)) bigram = probas[1] for token,value,bo in bigram: if token=="a b": self.assertEqual(round(value,6), round(math.log(0.466667,10),6)) if token=="b a": self.assertEqual(round(value,6), round(math.log(0.400000,10),6)) trigram = probas[2] for token,value,bo in trigram: if token=="a b a": self.assertEqual(round(value,6), round(math.log(0.142857,10),6)) if token==START_SENT_SYMBOL+"a a": self.assertEqual(round(value,6), round(math.log(0.500000,10),6)) if token=="a b"+END_SENT_SYMBOL: self.assertEqual(round(value,6), round(math.log(0.428571,10),6))