def NgramModel(sen): # Materi Syntatic proses:N-Gram # http://blog.pantaw.com/syntatic-proses-n-grams/ kata = TextTokenizer(sen) kata = ngram.ngrams(kata, n=2, njump=3) print "Total sample: ", len(kata)
def constructVocab(vect, totalword, nforgram, separate=True, njump=0): # kata masuk sudah ditokenize # kalo buat N-Gram brarti: ['saya sedang', 'sedang makan',...,'warung depan'] freqDist = dict() total_word = totalword dataset = [] jumpdataset = [] ############################### 2015-11-07 ################################ ## Dipisah, supaya penggunaan jump parameter tidak mempengaruhi ## jumlah dari keseluruhan kata, hanya akan mempengaruhi frekuensi sample ########################################################################### def scale(voc): for z in xrange(len(voc)): for y in xrange(len(voc[z])): # Untuk menangani bentuk Selain bigram, karena itu kita menggunakan format ini freqDist.setdefault(' '.join(voc[z][y]), 0) # C(Wi) => rekam jumlah kemunculan suatu kata freqDist[' '.join(voc[z][y])] += 1 for z in vect: # Semua yang masuk ke language model ngram, harus melalui proses tokenize->ngram if separate: if '<s>' not in z: z.insert(0, '<s>') if '</s>' not in z: z.insert(len(z), '</s>') dataset.append(ngram.ngrams(z, n=nforgram)) if njump > 0 and nforgram > 1: jumpdataset.append(ngram.ngrams(z, n=nforgram, njump=njump)) scale(dataset) if totalword == 0: # self.total_word = N = Jumlah(C) seluruh kata/sample pada kalimat/ruang sample total_word = functools.reduce( operator.add, freqDist.values()) #sum(freqDist.values()) if jumpdataset: scale(jumpdataset) print("Vocab size for N=%i is:%i, with Total Word(corpus length):%i" % (nforgram, len(freqDist), total_word)) return freqDist, total_word
def NgramModel(sen): # Materi Syntatic proses:N-Gram # http://blog.pantaw.com/syntatic-proses-n-grams/ kata = TextTokenizer(sen) kata = ngram.ngrams(kata,n=2,njump=3) print "Jumlah sample: ", len(kata) for z in kata: print ' '.join(z) print "\n"
def NgramModel(sen): # Materi Syntatic proses:N-Gram # http://blog.pantaw.com/syntatic-proses-n-grams/ kata = TextTokenizer(sen) kata = ngram.ngrams(kata,n=2,njump=3) print "Jumlah sample: ", len(kata) for z in kata: print ' '.join(z) print "\n"
def constructVocab(vect, totalword, nforgram, separate=True, njump=0): # kata masuk sudah ditokenize # kalo buat N-Gram brarti: ['saya sedang', 'sedang makan',...,'warung depan'] freqDist=dict() total_word=totalword dataset=[] jumpdataset=[] ############################### 2015-11-07 ################################ ## Dipisah, supaya penggunaan jump parameter tidak mempengaruhi ## jumlah dari keseluruhan kata, hanya akan mempengaruhi frekuensi sample ########################################################################### def scale(voc): for z in xrange(len(voc)): for y in xrange(len(voc[z])): # Untuk menangani bentuk Selain bigram, karena itu kita menggunakan format ini freqDist.setdefault(' '.join(voc[z][y]),0) # C(Wi) => rekam jumlah kemunculan suatu kata freqDist[' '.join(voc[z][y])]+=1 for z in vect: # Semua yang masuk ke language model ngram, harus melalui proses tokenize->ngram if separate: if '<s>' not in z: z.insert(0,'<s>') if '</s>' not in z: z.insert(len(z),'</s>') dataset.append(ngram.ngrams(z,n=nforgram)) if njump>0 and nforgram>1: jumpdataset.append(ngram.ngrams(z,n=nforgram,njump=njump)) scale(dataset) if totalword==0: # self.total_word = N = Jumlah(C) seluruh kata/sample pada kalimat/ruang sample total_word = functools.reduce(operator.add, freqDist.values())#sum(freqDist.values()) if jumpdataset: scale(jumpdataset) print ("Vocab size for N=%i is:%i, with Total Word(corpus length):%i" % (nforgram,len(freqDist),total_word)) return freqDist,total_word