Example #1
0
 def load_language_models(self):
     for x in listdir( STAT_DIR ):
         if x[-4:] == ".dat":
             modelfile = file(os.path.join(STAT_DIR,x))
             language = x[0:-4]
             self.languages.append( language )
             new_model = Trigrams()
             for line in modelfile:
                 tokens = split(line)
                 if len(tokens) == 2:
                     trigram = lower( unicode( tokens[0],'utf-8') )
                     probability = float(tokens[1])
                     new_model.add_trigram(trigram,probability)
             self.models.append(new_model)
             modelfile.close()