def TextTokenizer(sen): # Materi Syntatic proses:text tokenizing # http://blog.pantaw.com/syntatic-proses-text-tokenizing/ stopwords= ['kah','lah','pun','jah','jeh','mu','ku','ke','di','tapi','saya','kamu','mereka','dia', \ 'kita','adalah','dan','jika','kalau','sama','yang', \ 'sekarang','nanti','besok','kemarin','kemaren','nya','na',\ 'at','apa','ini','itu','juga','ketika','namun',\ 'sebab','oleh','malah','memang'] tok = tokenize() kata = tok.WordTokenize(sen,removepunct=False) if kata: print "kalimat setelah di tokenize: ", kata, "\n" return kata
def TextTokenizer(sen): # Materi Syntatic proses:text tokenizing # http://blog.pantaw.com/syntatic-proses-text-tokenizing/ stopwords= ['kah','lah','pun','jah','jeh','mu','ku','ke','di','tapi','saya','kamu','mereka','dia', \ 'kita','adalah','dan','jika','kalau','sama','yang', \ 'sekarang','nanti','besok','kemarin','kemaren','nya','na',\ 'at','apa','ini','itu','juga','ketika','namun',\ 'sebab','oleh','malah','memang'] tok = tokenize() kata = tok.WordTokenize(sen, removepunct=False) if kata: print "kalimat setelah di tokenize: ", kata, "\n" return kata
def rawForLangmodel(self, f, punct_remove=False, to_token=True, min_word=2): tok = tokenize() table = string.maketrans("","") # Splitting sentence based on new line then Regex pattern[0] words = re.split(r'\n',f) words = re.split(r''+pattern[0]+'',f) if punct_remove: words = [z.translate(table, string.punctuation) for z in words] if to_token: words = [tok.WordTokenize(z) for z in words] words = filter(lambda x: len(x) >= min_word, words) else: words = filter(lambda x: len(tok.WordTokenize(x)) >= min_word, words) return words
def rawForLangmodel(self, f, punct_remove=False, to_token=True, min_word=2): tok = tokenize() table = string.maketrans("","") # Splitting sentence based on new line then Regex pattern[0] words = re.split(r'\n',f) words = re.split(r''+pattern[0]+'',f) if punct_remove: words = [z.translate(table, string.punctuation) for z in words] if to_token: words = [tok.WordTokenize(z) for z in words] words = filter(lambda x: len(x) >= min_word, words) else: words = filter(lambda x: len(tok.WordTokenize(x)) >= min_word, words) return words
def rawForVector(self, f, min_word=2): """ Word level vector """ tok = tokenize() t0 = time() #print "Splitting sentence for vector processing..." table = string.maketrans("", "") words = re.split(r'' + pattern[0] + '', f) words = [z.translate(table, string.punctuation) for z in words] words = filter(lambda x: len(tok.WordTokenize(x)) >= min_word, words) words = [tok.WordTokenize(z) for z in words] #print "total sentence for process: ", len(words) #print "total unique words(vocabulary): ", len(self.word_constructor(words)) #print("Splitting sentence for vector done in %fs" % (time() - t0)) #print "\n" return words
def rawForVector(self, f, min_word=2): """ Word level vector """ tok = tokenize() t0 = time() #print "Splitting sentence for vector processing..." table = string.maketrans("","") words = re.split(r''+pattern[0]+'',f) words = [z.translate(table, string.punctuation) for z in words] words = filter(lambda x: len(tok.WordTokenize(x)) >= min_word, words) words = [tok.WordTokenize(z) for z in words] #print "total sentence for process: ", len(words) #print "total unique words(vocabulary): ", len(self.word_constructor(words)) #print("Splitting sentence for vector done in %fs" % (time() - t0)) #print "\n" return words
def train(self, vect, optimizer=None, separate=True, njump=0, verbose=False): """ In the study of probability, given at least two random variables X, Y, ...,that are defined on a probability space S, the joint probability distribution for X, Y, ... is a probability distribution that gives the probability that each of X, Y, ... falls in any particular range or discrete set of values specified for that variable ##################################################################################################################### ############################################################################################################# demikian jika diberikan kalimat S: "saya sedang makan nasi goreng di warung depan" berarti p(w1,w2,...,wn) disebut sebagai distribusi probabilitas(dimana w1,w2,...wn sebagai random variables), kata (w1,w2,...,wn), over the kalimat S """ t0 = time() print "Begin training language model..." if optimizer=='modkn': """NOTE: Untuk sementara penerapan njump parameter belum dapat digunakan dalam pengimplementasian Modified Kneser-Ney optimizer ini. """ print "Using optimizer: ", 'Modified Kneser-Ney' modkn = ModifiedKneserNey() modkn.kneser_ney_discounting(vect) modkn.train() #print "proba\t\ttoken\t\tbow\t\tcount" tmpN=1 for k,v in sorted(modkn.mKNeyEstimate.items(),key=lambda x: x[1][3]): #print ("%0.7f\t%s\t\t%0.7f\t\t%d"%(exp(v[0]),k,exp(v[1]),v[2])) if len(k.split(' ')) != tmpN: self.finalmodel[tmpN]=self.vocab self.vocab={} tmpN = len(k.split(' ')) self.vocab[k] = SimpleVocab(count=int(v[2]), estimator=v[0]) self.finalmodel[tmpN]=self.vocab del self.vocab else: if optimizer =='sgt': print "Using optimizer: ", 'Simple Good-Turing' elif optimizer == 'ls': print "Using optimizer: ", 'Laplace' else: print "Using optimizer: ",'Maximum Likelihood Estimation' tok = tokenize() for i in range(1,self.nforgram+1): self.nforgram=i self.raw_vocab,self.total_word = constructVocab(vect, self.total_word, \ nforgram=self.nforgram, separate=separate, \ njump=njump) if optimizer=='sgt': sgtN= float(functools.reduce(operator.add,self.raw_vocab.values())) sgt = SimpleGoodTuring(self.raw_vocab, sgtN) sgtSmoothProb,p0 = sgt.train(self.raw_vocab) for k, v in self.raw_vocab.iteritems(): # Hitung: # P(Wi) = C(Wi) / N <= untuk UniGram <= dicari melalui MLE if i==1: # Unigram do not use history """ WARNING!!! Kalau menggunakan SGT smoothing, MLE tidak digunakan """ if optimizer=='ls': V=len(self.raw_vocab) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,self.total_word,ls=True,V=V)) elif optimizer =='sgt': self.vocab[k] = SimpleVocab(count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,self.total_word)) elif i ==2: """ Perlu diingat motivasi dibalik NGram LM adalah: begin with the task of computing P(w|h), the probability of a word w given some history h """ # P(Wi, Wj) = C(Wi, Wj) / N <= untuk BiGram <= dicari melalui MLE # tetapi yang perlu kita cari adalah P(Wj | Wi) == conditional distribution untuk # seberapa kemungkinan kata Wj muncul diberikan kata Wi sebelumnya # P(Wj | Wi) = P(Wi, Wj) / P(Wi) = C(Wi, Wj) / C(Wi) # C(Wi)=> adalah unigram count CWi = self.finalmodel[i-1][tok.WordTokenize(k)[0]].count if optimizer=='ls': #functools.reduce(operator.add,self.raw_vocab.values()) V=len(self.raw_vocab) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,CWi,ls=True,V=V)) elif optimizer =='sgt': self.vocab[k] = SimpleVocab(count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,CWi)) elif i ==3: ####################################################################################### # P(Wi, Wj, Wk) = C (Wi, Wj, Wk) / N <= untuk Trigram # tetapi yang perlu kita cari adalah P(Wk | Wi, Wj) == conditional distribution untuk # seberapa kemungkinan kata Wk muncul diberikan kata Wi,Wj sebelumnya CWi = self.finalmodel[i-1][' '.join(tok.WordTokenize(k)[:-1])].count if optimizer=='ls': V=len(self.raw_vocab) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,CWi,ls=True,V=V)) elif optimizer =='sgt': self.vocab[k] = SimpleVocab(count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE(v,CWi)) self.finalmodel[i]=self.vocab self.vocab={} del self.raw_vocab self.perplexity(self.finalmodel, verbose=verbose) print ("Training language model done in %fs" % (time() - t0)) if verbose: print "token \t count \t proba \n", for k, v in self.finalmodel.iteritems(): print "######################################################################" print k, " - Gram", "\n", print "######################################################################" for ke,va in v.iteritems(): print ("%s\t %d\t %0.5f"%(ke,va.count,exp(va.estimator))) return self.finalmodel
def wordTokenizer(self, sent, simple=False): if not simple: tok = tokenize() return tok.WordTokenize(sent) else: return [x.strip() for x in re.split('(\W+)?', text) if x.strip()]
def train(self, vect, optimizer=None, separate=True, njump=0, verbose=False): """ In the study of probability, given at least two random variables X, Y, ...,that are defined on a probability space S, the joint probability distribution for X, Y, ... is a probability distribution that gives the probability that each of X, Y, ... falls in any particular range or discrete set of values specified for that variable ##################################################################################################################### ############################################################################################################# demikian jika diberikan kalimat S: "saya sedang makan nasi goreng di warung depan" berarti p(w1,w2,...,wn) disebut sebagai distribusi probabilitas(dimana w1,w2,...wn sebagai random variables), kata (w1,w2,...,wn), over the kalimat S """ t0 = time() print "Begin training language model..." if optimizer == 'modkn': """NOTE: Untuk sementara penerapan njump parameter belum dapat digunakan dalam pengimplementasian Modified Kneser-Ney optimizer ini. """ print "Using optimizer: ", 'Modified Kneser-Ney' modkn = ModifiedKneserNey() modkn.kneser_ney_discounting(vect) modkn.train() #print "proba\t\ttoken\t\tbow\t\tcount" tmpN = 1 for k, v in sorted(modkn.mKNeyEstimate.items(), key=lambda x: x[1][3]): #print ("%0.7f\t%s\t\t%0.7f\t\t%d"%(exp(v[0]),k,exp(v[1]),v[2])) if len(k.split(' ')) != tmpN: self.finalmodel[tmpN] = self.vocab self.vocab = {} tmpN = len(k.split(' ')) self.vocab[k] = SimpleVocab(count=int(v[2]), estimator=v[0]) self.finalmodel[tmpN] = self.vocab del self.vocab else: if optimizer == 'sgt': print "Using optimizer: ", 'Simple Good-Turing' elif optimizer == 'ls': print "Using optimizer: ", 'Laplace' else: print "Using optimizer: ", 'Maximum Likelihood Estimation' tok = tokenize() for i in range(1, self.nforgram + 1): self.nforgram = i self.raw_vocab,self.total_word = constructVocab(vect, self.total_word, \ nforgram=self.nforgram, separate=separate, \ njump=njump) if optimizer == 'sgt': sgtN = float( functools.reduce(operator.add, self.raw_vocab.values())) sgt = SimpleGoodTuring(self.raw_vocab, sgtN) sgtSmoothProb, p0 = sgt.train(self.raw_vocab) for k, v in self.raw_vocab.iteritems(): # Hitung: # P(Wi) = C(Wi) / N <= untuk UniGram <= dicari melalui MLE if i == 1: # Unigram do not use history """ WARNING!!! Kalau menggunakan SGT smoothing, MLE tidak digunakan """ if optimizer == 'ls': V = len( self.raw_vocab ) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, self.total_word, ls=True, V=V)) elif optimizer == 'sgt': self.vocab[k] = SimpleVocab( count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, self.total_word)) elif i == 2: """ Perlu diingat motivasi dibalik NGram LM adalah: begin with the task of computing P(w|h), the probability of a word w given some history h """ # P(Wi, Wj) = C(Wi, Wj) / N <= untuk BiGram <= dicari melalui MLE # tetapi yang perlu kita cari adalah P(Wj | Wi) == conditional distribution untuk # seberapa kemungkinan kata Wj muncul diberikan kata Wi sebelumnya # P(Wj | Wi) = P(Wi, Wj) / P(Wi) = C(Wi, Wj) / C(Wi) # C(Wi)=> adalah unigram count CWi = self.finalmodel[i - 1][tok.WordTokenize(k)[0]].count if optimizer == 'ls': #functools.reduce(operator.add,self.raw_vocab.values()) V = len( self.raw_vocab ) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, CWi, ls=True, V=V)) elif optimizer == 'sgt': self.vocab[k] = SimpleVocab( count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, CWi)) elif i == 3: ####################################################################################### # P(Wi, Wj, Wk) = C (Wi, Wj, Wk) / N <= untuk Trigram # tetapi yang perlu kita cari adalah P(Wk | Wi, Wj) == conditional distribution untuk # seberapa kemungkinan kata Wk muncul diberikan kata Wi,Wj sebelumnya CWi = self.finalmodel[i - 1][' '.join( tok.WordTokenize(k)[:-1])].count if optimizer == 'ls': V = len( self.raw_vocab ) #<= gunakan jika menggunakan laplace smoothing self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, CWi, ls=True, V=V)) elif optimizer == 'sgt': self.vocab[k] = SimpleVocab( count=v, estimator=sgtSmoothProb[k]) else: self.vocab[k] = SimpleVocab(count=v, estimator=self.MLE( v, CWi)) self.finalmodel[i] = self.vocab self.vocab = {} del self.raw_vocab self.perplexity(self.finalmodel, verbose=verbose) print("Training language model done in %fs" % (time() - t0)) if verbose: print "token \t count \t proba \n", for k, v in self.finalmodel.iteritems(): print "######################################################################" print k, " - Gram", "\n", print "######################################################################" for ke, va in v.iteritems(): print("%s\t %d\t %0.5f" % (ke, va.count, exp(va.estimator))) return self.finalmodel