def run_crf(trainfile, testfile, model_file=None): maxlen = 100 sents_train, tags_train, unique_words_train, unique_tags_train = \ P.retrieve_sentences_tags(trainfile, maxlen=maxlen) sents_test, tags_test, unique_word_test, unique_tags_test = \ P.retrieve_sentences_tags(testfile, maxlen=maxlen, allowedtags=unique_tags_train) train_data = [] for n, st in enumerate(sents_train): s = [] for m, _ in enumerate(st): s.append((unicode(sents_train[n][m], "utf-8") , unicode(tags_train[n][m], "utf-8"))) train_data.append(s) crf = CRFTagger() if model_file is None: crf.train(train_data, model_file='data/crf.mdl') else: crf.set_model_file(model_file) test_data = [] for n, st in enumerate(sents_test): s = [] for m, _ in enumerate(st): s.append((unicode(sents_test[n][m], "utf-8") , unicode(tags_test[n][m], "utf-8"))) test_data.append(s) print(crf.evaluate(test_data))
def tag_crf(self, untagged_string: str): """Tag POS with CRF tagger. :type untagged_string: str :param : An untagged, untokenized string of text. :rtype tagged_text: str """ untagged_tokens = wordpunct_tokenize(untagged_string) pickle_path = self.available_taggers['crf'] tagger = CRFTagger() tagger.set_model_file(pickle_path) tagged_text = tagger.tag(untagged_tokens) return tagged_text
def ExtractItemsFromJudgment(text,CodeTaggerFile,TitleTaggerFile): text = removeHTMLTags(text) tokenList = tokenizeTestData(text) CodesTagger = CRFTagger() TitleTagger = CRFTagger() CodesTagger.set_model_file(CodeTaggerFile) TitleTagger.set_model_file(TitleTaggerFile) taggedCodes = CodesTagger.tag_sents(tokenList) taggedTitles = TitleTagger.tag_sents(tokenList) return extract_entities(taggedCodes,taggedTitles)
def ExtractItemsFromJudgment(text): text = removeHTMLTags(text) tokenList = tokenizeTestData(text) CodesTagger = CRFTagger() titleTagger = CRFTagger() CodesTagger.set_model_file("models/CRF-Model-OnlyCodes") titleTagger.set_model_file("models/CRF-Model-OnlyTitles") taggedCodes = CodesTagger.tag_sents(tokenList) taggedTitles = titleTagger.tag_sents(tokenList) return extract_entities(taggedCodes,taggedTitles)
def chunking(sents, chunked_file): ''' Chunking param sents: 列表,如[['dog', 'is', 'dog'], ['dog', 'good']] ''' os.chdir('/home/zqr/code/chunk2vec/') start_time = time.time() #PoS print '\n-->Start PoS' #print '->Training PoS Tagger' #ct = CRFTagger() #ct.train(chunk_traindata(pos_trainfile), 'model.crf.tagger') #print '->Done' #pos_testdata_gold = chunk_traindata(pos_testfile) # pos corpus print '->Load CRF Tagger model' ct = CRFTagger() ###这个model是从chunk任务中学习到的PoS标签 ct.set_model_file('model.crf.tagger') print '->Posing' tagged_sents = ct.tag_sents(sents) #print 'PoS acc.:', ct.evaluate(pos_testdata_gold) #将PoS好的句子写文件 print '->Write posed file' pos_data(tagged_sents, 'tmp_for_chunking') end_time = time.time() print '-->Done, Time:', end_time - start_time, 's' #节省时间,暂时用测试语料 #pos_data(pos_testdata_gold, chunk_inputfile) start_time = time.time() ###Chunk,依赖系统安装YamCha,训练语料就用CoNLL的训练语料 print '\n-->Start Chunking' os.system('yamcha-config --libexecdir') #os.chdir('/home/zqr/code/sent2vec/') os.system('cp /home/zqr/local/libexec/yamcha/Makefile .') #训练chunking模型 #os.system('make CORPUS=' + pos_trainfile +' MODEL=chunk_model train') os.system('yamcha -m chunk_model.model < tmp_for_chunking > ' + chunked_file) print '-->Done, Time:', time.time() - start_time, 's'
from flask import Flask, request, jsonify import os import nltk from nltk.tag import CRFTagger import numpy as np app = Flask(__name__) ct = CRFTagger() ct.set_model_file( os.path.dirname(os.path.abspath(__file__)) + '/all_indo_man_tag_corpus_model.crf.tagger') @app.route('/', methods=['POST']) def process(): # Tokenize input text input_text = nltk.word_tokenize(request.form.get('input', '')) # Tag sentence result = ct.tag_sents([input_text]) # Remove unwanted elements forbidden_tags = ['SC', 'IN', 'CC'] for index, sentence in enumerate(result): result[index] = [ word for word in sentence if word[1] not in forbidden_tags ] # Assemble output output = ''
def postag_sequence(self, data): ct = CRFTagger() ct.set_model_file(POSTAG_MODEL_DIR) data["postag_seq"] = ct.tag_sents([data["preprocessed_kalimat"]])[0]
# "CHARGE": "CHARGE", # "APERTURE": "APERTURE", # "STRENGTH": "STRENGTH", # "FRAGRANT": "FRAGRANT", # "KEY TYPE": "KEY TYPE", # "KEY BRAND": "KEY BRAND", # "KEY_TYPE": "KEY_TYPE", # "KEY_BRAND": "KEY_BRAND", # "KEY_NAME": "KEY_NAME", # "KEY_OS": "KEY_OS", } TAGGER3 = CRFTagger() TAGGER3.set_model_file( os.path.abspath( 'server/nlp/data/all_indo_man_tag_corpus_model.crf.tagger')) def getPOSTag(_temporary_tokens): strin = [] for token_tag in _temporary_tokens: if token_tag[0].encode('ascii', 'ignore').decode('utf8'): strin.append(token_tag[0].encode('ascii', 'ignore').decode('utf8')) return [(str(token.encode('ascii', 'ignore'), 'utf8'), str(tag.encode('ascii', 'ignore'), 'utf8')) for (token, tag) in TAGGER3.tag_sents([strin])[0]] def getPOSTagTesting(_temporary_tokens):
try: import pycrfsuite except ImportError: pass print "ga ketemu" MAP_ENTITY_TAG = { "ORGANIZATION": "organization", "LOCATION": "location", "PERSON": "person", "TIME": "time", "QUANTITY": "quantity" } TAGGER3 = CRFTagger() TAGGER3.set_model_file('data/all_indo_man_tag_corpus_model.crf.tagger') def getPOSTag(_temporary_tokens): strin = [] for token_tag in _temporary_tokens: strin.append(unicode(token_tag[0].decode('utf-8'))) return [(token.encode('ascii', 'ignore'), tag.encode('ascii', 'ignore')) for (token, tag) in TAGGER3.tag_sents([strin])[0]] def parseEntityName(_sent): def getTypeData(_ne): """ ekstrak jenis Name Entity """
def onsentencelist(): ct = CRFTagger() """nertweetlist contains ner-tagged tweets""" nertweetlist = pickle.load(open("nertweetlist.pickle", "rb")) print(sorted(nertweetlist)[0:5]) print(len(nertweetlist)) """tweetlist contains the plain tweets """ tweetlist = pickle.load(open('tweetlist.pickle', 'rb')) print(len(tweetlist)) print(sorted(tweetlist)[0:5]) """training size as percentage""" trainingsize = 0.9 """ calculate where to split data """ limit = round(trainingsize * len(nertweetlist)) """train the data / choose one of the 2 blocks """ #train_data = nertweetlist[:limit] #ct.train(train_data,'tweetmodel.crf.tagger') ct.set_model_file('tweetmodel.crf.tagger') """Test data and evaluate""" test_data = tweetlist[limit:] ct.tag_sents(test_data) # tagging sentences gold_sentences = nertweetlist[limit:] print("\nAccuracy:", ct.evaluate(gold_sentences)) """ TURN TRAINED TAGGED LIST AND TEST LIST INTO ONE LIST CONTAINING ONLY THE TRUE AND PREDTAGS""" pred_nerlist = [] for sentence in tweetlist[:limit]: #print("DIT:", sentence) for (word, nertag) in ct.tag(sentence): pred_nerlist.append(nertag.lower()) true_nerlist = [] #ct_true = gold_sentences for sentence in nertweetlist[:limit]: for (word, nertag) in sentence: #true_nerlist.append((word,nertag)) true_nerlist.append(nertag.lower()) """ Print baseline """ #TODO: calulate baseline """"Print F-score and confusion matrix """ print("\nF-score (micro):", f1_score(true_nerlist, pred_nerlist, average='micro')) print("\nF-score (macro):", f1_score(true_nerlist, pred_nerlist, average='macro')) print("\nF-score (weigthed):", f1_score(true_nerlist, pred_nerlist, average='weighted')) print( "\nF-score (None):", f1_score(true_nerlist, pred_nerlist, average=None, labels=[ "o", "b-per", "i-per", "b-loc", "i-loc", "b-org", "i-org", "b-misc", "i-misc" ])) print("\nConfusion matrix:\n") for item in [ "O", "B-per", "I-per", "B-loc", "I-loc", "B-org", "I-org", "B-misc", "I-misc" ]: print(" ", item, end="") print( "\n", confusion_matrix(true_nerlist, pred_nerlist, labels=[ "o", "b-per", "i-per", "b-loc", "i-loc", "b-org", "i-org", "b-misc", "i-misc" ]))
def get_sentimen(doc): ct = CRFTagger() ct.set_model_file('all_indo_man_tag_corpus_model.crf.tagger') # doc=remove_hashtag(doc) # doc=clean_tweet(doc) doc = utils.cleanAllTweet(doc) #print(doc) #pisah perkalimat sentences = nltk.sent_tokenize(doc) #pisah per kata stokens = [nltk.word_tokenize(sent) for sent in sentences] #tag indonesia taggedlist = ct.tag_sents(stokens) # print(taggedlist) #sentiword Indonesia barasa = pd.read_csv( 'barasa-ID.txt', delimiter='\t', encoding='utf-8', header=0, names=['syn', 'goodness', 'lemma', 'pos', 'neg', 'dummy']) score_list = [] negasi = "" for idx, taggedsent in enumerate(taggedlist): score_list.append([]) for idx2, t in enumerate(taggedsent): newtag = '' if t[1].startswith('NN'): newtag = 'n' elif t[1].startswith('JJ'): newtag = 'a' elif t[1].startswith('VB'): newtag = 'v' elif t[1].startswith('R'): newtag = 'r' elif t[1].startswith('NEG'): negasi = t[0] else: newtag = '' if (newtag != ''): if (negasi != ""): kalimat = negasi + ' ' + t[0] negasi = "" else: kalimat = t[0] lemmas = barasa[barasa['lemma'].str.contains(kalimat, na=False)] score_list[idx].append(get_scores(lemmas)) #sentence_sentiment=[] totalscore = 0.0 for score_sent in score_list: scoresentnow = sum([word_score for word_score in score_sent]) / len(score_sent) #sentence_sentiment.append(score_sent) totalscore = scoresentnow + totalscore sentimenScore = totalscore / len(score_list) print("Score sentimen = " + str(sentimenScore)) return sentimenScore
# -*- coding: utf-8 -*- from nltk.tag import CRFTagger from pythainlp.tokenize import word_tokenize ct = CRFTagger() ct.set_model_file('model.crf.tagger') text = "" while text != "exit": text = input("Text : ") post = word_tokenize(text, 'icu') print(ct.tag_sents([post]))
class Chunker: UNIQ = '_UNIQUE_STRING_' CHUNK_PARSER = None """ """ def __init__(self): # Memuat data pre-trained POS-Tagger uni, bi, tri, word = self.load_obj("tagger") self.TAGGER1 = Tagger(uni, bi, tri, word) # Memuat data pre-trained POS-Tagger uni2, bi2, tri2, word2 = self.load_obj("tagger2") self.TAGGER2 = Tagger(uni2, bi2, tri2, word2) self.TAGGER3 = CRFTagger() self.TAGGER3.set_model_file( 'dataset/all_indo_man_tag_corpus_model.crf.tagger') # Memuat data grammar chunker self.load_chunker() """ """ def load_obj(self, name): with open('obj/' + name + '.pkl', 'rb') as f: return pickle.load(f) """ Melakukan formatting string menjadi regex """ def format_to_re(self, format): parts = (format % MarkPlaceholders()).split(self.UNIQ) for i in range(0, len(parts), 2): parts[i] = re.escape(parts[i]) return ' '.join(parts).replace('\\', '') """ Mengubah tree POS Tag menjadi tree chunk """ def tree_to_str(self, tree_data): ne_in_sent = [] for subtree in tree_data: if type(subtree ) == Tree: # If subtree is a noun chunk, i.e. NE != "O" ne_label = subtree.label() ne_string = " ".join( [token for token, pos in subtree.leaves()]) ne_in_sent.append((ne_string, ne_label)) else: ne_in_sent.append((subtree[0], subtree[1])) return ne_in_sent """ Memuat rule chunk """ def load_chunker(self): try: f = open('dataset/phrase_chunker_grammar_id.txt') files = self.format_to_re(f.read()) grammars = files f.close() self.CHUNK_PARSER = nltk.RegexpParser(grammars) except Exception as e: print str(e) """ Mengubah tree chunk menjadi list of chunk dalam bentuk list of string """ def get_only_str(self, tree_chunk): output = [] for chunk, tag in tree_chunk: output.append(chunk) return output """ Mengubah list of chunk(string) menjadi string dengan format: [chunk1] [chunk2] ... [chunkN] """ def beautify(self, chunks): strout = "" for s in chunks: strout += "[" + s + "] " return strout """ Memberi POSTag pada setiap kata pada kalimat Melakukan chunking kalimat Mengembalikan chunk Tree """ def chunk_me1(self, _str): return self.CHUNK_PARSER.parse( self.TAGGER1.tagSentence(_str.split(" "))) """ Memberi POSTag pada setiap kata pada kalimat Melakukan chunking kalimat Mengembalikan chunk Tree """ def chunk_me2(self, _str): return self.CHUNK_PARSER.parse( self.TAGGER2.tagSentence(_str.split(" "))) """ """ def chunk_me3(self, _str): _strs = _str.split(" ") strs = [] for s in _strs: strs.append(unicode(s)) return self.CHUNK_PARSER.parse(self.TAGGER3.tag_sents([strs])[0])
# test_sents = [[ele[0] for ele in sent] for sent in test_sents] # from dpattack.libs.luna import time_record # with time_record(): # for i in range(2048): # tagger.tag_sents(test_sents[i:i+1]) # crf 0.915 seconds 0.723 0.526 0.283 # with time_record(): # tagger.tag_sents(test_sents[:2048]) # crf 0.936 seconds 0.661 0.495 0.278 # tagger = nltk.BigramTagger([[('the', 'dt'), ('work', 'nn'), ('of', 'in')]]) # print(tagger.tag_sents([('the', 'end', 'of')])) sent = [ "at <UNK> p.m. , at the throw of the `` cooling off '' period , the average was down <UNK> points .".split(" ")] tagger = CRFTagger() tagger.set_model_file( "/disks/sdb/zjiehang/zhou_data/saved_models/crftagger") print(tagger.tag_sents(sent)) from dpattack.libs.luna import auto_create tagger = auto_create("trigram_tagger", lambda: train_gram_tagger( train_corpus, ngram=3), cache=True, path='/disks/sdb/zjiehang/zhou_data/saved_vars') print(tagger.tag_sents(sent)) def gen_tag_dict(corpus: Corpus, vocab: Vocab, threshold=3, verbose=True): """ Rule:
"PRON VERB NOUN VERB ADP DET NOUN") """ # Extract features from words in the given text features = generateUtterancesFeatures(text) # Predict tags for the given utterance tags = crf.predict(features) return ' '.join(str(t) for w in tags for t in w) # CRF POS TAGGING - PRE-TRAINED POS-TAGGER # Path to the pre-trained POS-tagger TAGGER_PATH = "crfpostagger" # Initialize tagger tagger = CRFTagger() tagger.set_model_file(TAGGER_PATH) #def tag_text_CRF(text): # """ # Function used for tagging the given text. This function uses a CRF predefined pos tagger. # # :param text: text to be associated with the POS tags # :return: string containing POS tags for the given text; each tag refers to the word at the same index in the sentence # (eg. for the sentence "My dog likes running around the garden." the returned string with tags is # "PRP$ NN VBZ VBG IN DT NN") # """ # # tags= tagger.tag([word.lower() for word in text.split()]) # return ' '.join(str(t) for w,t in tags) # HMM POS TAGGING
from nltk.tag import CRFTagger ct = CRFTagger() ct.set_model_file('pos-tagger-indonesia-model.tagger') hasil = ct.tag_sents([['Saya', 'bekerja', 'di', 'Bandung'], ['Nama', 'saya', 'Yudi']]) print(hasil)
#!/usr/bin/env python # -*- coding: utf-8 -*- """Readers for the pke module.""" import xml.etree.ElementTree as etree import spacy from pke.data_structures import Document from nltk.tag import CRFTagger from nltk.tokenize import sent_tokenize, word_tokenize, TweetTokenizer from nltk.corpus import stopwords import string from Sastrawi.Stemmer.StemmerFactory import StemmerFactory ct = CRFTagger() ct.set_model_file('./all_indo_man_tag_corpus_model.crf copy.tagger') factory = StemmerFactory() stemmer = factory.create_stemmer() tokenizer_words = TweetTokenizer() class Reader(object): def read(self, path): raise NotImplementedError class MinimalCoreNLPReader(Reader): """Minimal CoreNLP XML Parser.""" def __init__(self): self.parser = etree.XMLParser() def read(self, path, **kwargs):
from nltk.tag import CRFTagger import nltk import credentials_var as cred ct = CRFTagger() ct.set_model_file('../references/all_indo_man_tag_corpus_model.crf.tagger') def pos_tagger(tokens): return ct.tag_sents([tokens]) for element in list(cred.find_all): text = element['extended_tweet']['full_text'] if element['truncated'] is True else element['text'] print(pos_tagger(nltk.tokenize.word_tokenize(text)))
from nltk.tag import CRFTagger crflan = CRFTagger() crf = CRFTagger() crflan.set_model_file('model.crf.tagger') crf.set_model_file('model1.crf.tagger') print "Give a sentence..." # Test test_sent = raw_input() test_sent = test_sent.encode('utf-8').decode('utf-8').split(' ') print test_sent half_ans = crflan.tag(test_sent) print half_ans # print test_sent print crf.tag(test_sent)
prefix = [] for word, pos in zip(word_pos_data[speaker][1], word_pos_data[speaker][2]): prefix.append(word.replace("$unc$", "")) sp_data.append((unicode(word.replace("$unc$", "") .encode("utf8")), unicode(pos.encode("utf8")))) training_data.append(deepcopy(sp_data)) print "training tagger..." ct.train(training_data, TAGGER_PATH) if TEST: print "testing tagger..." ct = CRFTagger() # initialize tagger ct.set_model_file(TAGGER_PATH) dialogue_speakers = [] for disf_file in DISFLUENCY_TEST_FILES: IDs, mappings, utts, pos_tags, labels = \ load_data_from_disfluency_corpus_file(disf_file) dialogue_speakers.extend(sort_into_dialogue_speakers(IDs, mappings, utts, pos_tags, labels)) word_pos_data = {} # map from the file name to the data for data in dialogue_speakers: dialogue, a, b, c, d = data word_pos_data[dialogue] = (a, b, c, d) ct.tag([unicode(w) for w in "uh my name is john".split()]) # either gather training data or test data
# ---------------------------------------------------------------| stop_words = set(stopwords.words('indonesian')) words = word_tokenize(text) new_sentence = [] for word in words: if word not in stop_words: new_sentence.append(word) new_sentence = [ unicode(new_sentence[x], "utf-8") for x in range(len(new_sentence)) ] ct = CRFTagger() ct.set_model_file('all_indo_man_tag_corpus_model.crf.tagger') hasil = ct.tag_sents([new_sentence]) # ---------------------------------------------------------------| # untuk melihat frekuensi kata yang muncul # ---------------------------------------------------------------| # fdist = FreqDist(new_sentence) # print(fdist.most_common()) # ---------------------------------------------------------------| # untuk melihat frekuensi kata yang muncul # ---------------------------------------------------------------| for tokenTag in hasil[0]: token, tag = tokenTag token_text = unicodedata.normalize(u'NFKD', token).encode(u'ascii', u'ignore')
from nltk.tag import CRFTagger from nltk.corpus import brown from sklearn.metrics import classification_report as crf ct = CRFTagger() ct.set_model_file("model.crf.tagger") brown_sents = brown.sents() size = int(len(brown_sents) * 0.7) test_sents = brown_sents[size:] flat_list = [] for sublist in test_sents: for item in sublist: flat_list.append(item) l = ct.tag(flat_list) y_pred = [] for each in l: y_pred.append(each[1]) #print(y_pred[:10]) tagged_sents = brown.tagged_sents(tagset="universal")[size:] y_true = [] for each in tagged_sents: for e in each:
class DeepDisfluencyTagger(IncrementalTagger): """A deep-learning driven incremental disfluency tagger (and optionally utterance-segmenter). Tags each word with the following: <f/> - a fluent word <e/> - an edit term word, not necessarily inside a repair structure <rms id="N"/> - reparandum start word for repair with ID number N <rm id="N"/> - mid-reparandum word for repair N <i id="N"/> - interregnum word for repair N <rps id="N"/> - repair onset word for repair N <rp id="N"/> - mid-repair word for repair N <rpn id="N"/> - repair end word for substitution or repetition repair N <rpnDel id="N"/> - repair end word for a delete repair N If in joint utterance segmentation mode according to the config file, the following utterance segmentation tags are used: <cc/> - a word which continues the current utterance and whose following word will continue it <ct/> - a word which continues the current utterance and is the last word of it <tc/> - a word which is the beginning of an utterance and whose following word will continue it <tt/> - a word constituting an entire utterance """ def __init__(self, config_file=None, config_number=None, saved_model_dir=None, pos_tagger=None, language_model=None, pos_language_model=None, edit_language_model=None, timer=None, timer_scaler=None, use_timing_data=False): if not config_file: config_file = os.path.dirname(os.path.realpath(__file__)) +\ "/../experiments/experiment_configs.csv" config_number = 35 print "No config file, using default", config_file, config_number super(DeepDisfluencyTagger, self).__init__(config_file, config_number, saved_model_dir) print "Processing args from config number {} ...".format(config_number) self.args = process_arguments(config_file, config_number, use_saved=False, hmm=True) # separate manual setting setattr(self.args, "use_timing_data", use_timing_data) print "Intializing model from args..." self.model = self.init_model_from_config(self.args) # load a model from a folder if specified if saved_model_dir: print "Loading saved weights from", saved_model_dir self.load_model_params_from_folder(saved_model_dir, self.args.model_type) else: print "WARNING no saved model params, needs training." print "Loading original embeddings" self.load_embeddings(self.args.embeddings) if pos_tagger: print "Loading POS tagger..." self.pos_tagger = pos_tagger elif self.args.pos: print "No POS tagger specified,loading default CRF switchboard one" self.pos_tagger = CRFTagger() tagger_path = os.path.dirname(os.path.realpath(__file__)) +\ "/../feature_extraction/crfpostagger" self.pos_tagger.set_model_file(tagger_path) if self.args.n_language_model_features > 0 or \ 'noisy_channel' in self.args.decoder_type: print "training language model..." self.init_language_models(language_model, pos_language_model, edit_language_model) if timer: print "loading timer..." self.timing_model = timer self.timing_model_scaler = timer_scaler else: # self.timing_model = None # self.timing_model_scaler = None print "No timer specified, using default switchboard one" timer_path = os.path.dirname(os.path.realpath(__file__)) +\ '/../decoder/timing_models/' + \ 'LogReg_balanced_timing_classifier.pkl' with open(timer_path, 'rb') as fid: self.timing_model = cPickle.load(fid) timer_scaler_path = os.path.dirname(os.path.realpath(__file__)) +\ '/../decoder/timing_models/' + \ 'LogReg_balanced_timing_scaler.pkl' with open(timer_scaler_path, 'rb') as fid: self.timing_model_scaler = cPickle.load(fid) # TODO a hack # self.timing_model_scaler.scale_ = \ # self.timing_model_scaler.std_.copy() print "Loading decoder..." hmm_dict = deepcopy(self.tag_to_index_map) # add the interegnum tag if "disf" in self.args.tags: intereg_ind = len(hmm_dict.keys()) interreg_tag = \ "<i/><cc/>" if "uttseg" in self.args.tags else "<i/>" hmm_dict[interreg_tag] = intereg_ind # add the interregnum tag # decoder_file = os.path.dirname(os.path.realpath(__file__)) + \ # "/../decoder/model/{}_tags".format(self.args.tags) noisy_channel = None if 'noisy_channel' in self.args.decoder_type: noisy_channel = SourceModel(self.lm, self.pos_lm, uttseg=self.args.do_utt_segmentation) self.decoder = FirstOrderHMM( hmm_dict, markov_model_file=self.args.tags, timing_model=self.timing_model, timing_model_scaler=self.timing_model_scaler, constraint_only=True, noisy_channel=noisy_channel) # getting the states in the right shape self.state_history = [] self.softmax_history = [] # self.convert_to_output_tags = get_conversion_method(self.args.tags) self.reset() def init_language_models(self, language_model=None, pos_language_model=None, edit_language_model=None): clean_model_dir = os.path.dirname(os.path.realpath(__file__)) +\ "/../data/lm_corpora" if language_model: self.lm = language_model else: print "No language model specified, using default switchboard one" lm_corpus_file = open(clean_model_dir + "/swbd_disf_train_1_clean.text") lines = [ line.strip("\n").split(",")[1] for line in lm_corpus_file if "POS," not in line and not line.strip("\n") == "" ] split = int(0.9 * len(lines)) lm_corpus = "\n".join(lines[:split]) heldout_lm_corpus = "\n".join(lines[split:]) lm_corpus_file.close() self.lm = KneserNeySmoothingModel( order=3, discount=0.7, partial_words=self.args.partial_words, train_corpus=lm_corpus, heldout_corpus=heldout_lm_corpus, second_corpus=None) if pos_language_model: self.pos_lm = pos_language_model elif self.args.pos: print "No pos language model specified, \ using default switchboard one" lm_corpus_file = open(clean_model_dir + "/swbd_disf_train_1_clean.text") lines = [ line.strip("\n").split(",")[1] for line in lm_corpus_file if "POS," in line and not line.strip("\n") == "" ] split = int(0.9 * len(lines)) lm_corpus = "\n".join(lines[:split]) heldout_lm_corpus = "\n".join(lines[split:]) lm_corpus_file.close() self.pos_lm = KneserNeySmoothingModel( order=3, discount=0.7, partial_words=self.args.partial_words, train_corpus=lm_corpus, heldout_corpus=heldout_lm_corpus, second_corpus=None) if edit_language_model: self.edit_lm = edit_language_model else: edit_lm_corpus_file = open(clean_model_dir + "/swbd_disf_train_1_edit.text") edit_lines = [ line.strip("\n").split(",")[1] for line in edit_lm_corpus_file if "POS," not in line and not line.strip("\n") == "" ] edit_split = int(0.9 * len(edit_lines)) edit_lm_corpus = "\n".join(edit_lines[:edit_split]) heldout_edit_lm_corpus = "\n".join(edit_lines[edit_split:]) edit_lm_corpus_file.close() self.edit_lm = KneserNeySmoothingModel( train_corpus=edit_lm_corpus, heldout_corpus=heldout_edit_lm_corpus, order=2, discount=0.7) # TODO an object for getting the lm features incrementally # in the language model def init_model_from_config(self, args): # for feat, val in args._get_kwargs(): # print feat, val, type(val) if not test_if_using_GPU(): print "Warning: not using GPU, might be a bit slow" print "\tAdjust Theano config file ($HOME/.theanorc)" print "loading tag to index maps..." label_path = os.path.dirname(os.path.realpath(__file__)) +\ "/../data/tag_representations/{}_tags.csv".format(args.tags) word_path = os.path.dirname(os.path.realpath(__file__)) +\ "/../data/tag_representations/{}.csv".format(args.word_rep) pos_path = os.path.dirname(os.path.realpath(__file__)) +\ "/../data/tag_representations/{}.csv".format(args.pos_rep) self.tag_to_index_map = load_tags(label_path) self.word_to_index_map = load_tags(word_path) self.pos_to_index_map = load_tags(pos_path) self.model_type = args.model_type vocab_size = len(self.word_to_index_map.keys()) emb_dimension = args.emb_dimension n_hidden = args.n_hidden n_extra = args.n_language_model_features + args.n_acoustic_features n_classes = len(self.tag_to_index_map.keys()) self.window_size = args.window n_pos = len(self.pos_to_index_map.keys()) update_embeddings = args.update_embeddings lr = args.lr print "Initializing model of type", self.model_type, "..." if self.model_type == 'elman': model = Elman(ne=vocab_size, de=emb_dimension, nh=n_hidden, na=n_extra, n_out=n_classes, cs=self.window_size, npos=n_pos, update_embeddings=update_embeddings) self.initial_h0_state = model.h0.get_value() self.initial_c0_state = None elif self.model_type == 'lstm': model = LSTM(ne=vocab_size, de=emb_dimension, n_lstm=n_hidden, na=n_extra, n_out=n_classes, cs=self.window_size, npos=n_pos, lr=lr, single_output=True, cost_function='nll') self.initial_h0_state = model.h0.get_value() self.initial_c0_state = model.c0.get_value() else: raise NotImplementedError('No model init for {0}'.format( self.model_type)) return model def load_model_params_from_folder(self, model_folder, model_type): if model_type in ["lstm", "elman"]: self.model.load_weights_from_folder(model_folder) self.initial_h0_state = self.model.h0.get_value() if model_type == "lstm": self.initial_c0_state = self.model.c0.get_value() else: raise NotImplementedError( 'No weight loading for {0}'.format(model_type)) def load_embeddings(self, embeddings_name): # load pre-trained embeddings embeddings_dir = os.path.dirname(os.path.realpath(__file__)) +\ "/../embeddings/" pretrained = gensim.models.Word2Vec.load(embeddings_dir + embeddings_name) print "emb shape", pretrained[pretrained.index2word[0]].shape # print pretrained[0].shape # assign and fill in the gaps emb = populate_embeddings(self.args.emb_dimension, len(self.word_to_index_map.items()), self.word_to_index_map, pretrained) self.model.load_weights(emb=emb) def standardize_word_and_pos( self, word, pos=None, proper_name_pos_tags=["NNP", "NNPS", "CD", "LS", "SYM", "FW"]): word = word.lower() if not pos and self.pos_tagger: pos = self.pos_tagger.tag([]) # TODO if pos: pos = pos.upper() if pos in proper_name_pos_tags and "$unc$" not in word: word = "$unc$" + word if self.pos_to_index_map.get(pos) is None: # print "unknown pos", pos pos = "<unk>" if self.word_to_index_map.get(word) is None: # print "unknown word", word word = "<unk>" return word, pos def tag_new_word(self, word, pos=None, timing=None, extra=None, diff_only=True, rollback=0): """Tag new incoming word and update the word and tag graphs. :param word: the word to consume/tag :param pos: the POS tag to consume/tag (optional) :param timing: the duration of the word (optional) :param diff_only: whether to output only the diffed suffix, if False, outputs entire output tags :param rollback: the number of words to rollback in the case of changed word hypotheses from an ASR """ self.rollback(rollback) if pos is None and self.args.pos: # if no pos tag provided but there is a pos-tagger, tag word test_words = [ unicode(x) for x in get_last_n_features( "words", self.word_graph, len(self.word_graph) - 1, n=4) ] + [unicode(word.lower())] pos = self.pos_tagger.tag(test_words)[-1][1] # print "tagging", word, "as", pos # 0. Add new word to word graph word, pos = self.standardize_word_and_pos(word, pos) # print "New word:", word, pos self.word_graph.append((word, pos, timing)) # 1. load the saved internal rnn state # TODO these nets aren't (necessarily) trained statefully # The internal state in training self.args.bs words back # are the inital ones in training, however here # They are the actual state reached. if self.state_history == []: c0_state = self.initial_c0_state h0_state = self.initial_h0_state else: if self.model_type == "lstm": c0_state = self.state_history[-1][0][-1] h0_state = self.state_history[-1][1][-1] elif self.model_type == "elman": h0_state = self.state_history[-1][-1] if self.model_type == "lstm": self.model.load_weights(c0=c0_state, h0=h0_state) elif self.model_type == "elman": self.model.load_weights(h0=h0_state) else: raise NotImplementedError("no history loading for\ {0} model".format(self.model_type)) # 2. do the softmax output with converted inputs word_window = [ self.word_to_index_map[x] for x in get_last_n_features("words", self.word_graph, len(self.word_graph) - 1, n=self.window_size) ] pos_window = [ self.pos_to_index_map[x] for x in get_last_n_features("POS", self.word_graph, len(self.word_graph) - 1, n=self.window_size) ] # print "word_window, pos_window", word_window, pos_window if self.model_type == "lstm": h_t, c_t, s_t = self.model.\ soft_max_return_hidden_layer([word_window], [pos_window]) self.softmax_history.append(s_t) if len(self.state_history) == 20: # just saving history self.state_history.pop(0) # pop first one self.state_history.append((c_t, h_t)) elif self.model_type == "elman": h_t, s_t = self.model.soft_max_return_hidden_layer([word_window], [pos_window]) self.softmax_history.append(s_t) if len(self.state_history) == 20: self.state_history.pop(0) # pop first one self.state_history.append(h_t) else: raise NotImplementedError("no softmax implemented for\ {0} model".format(self.model_type)) softmax = np.concatenate(self.softmax_history) # 3. do the decoding on the softmax if "disf" in self.args.tags: edit_tag = "<e/><cc>" if "uttseg" in self.args.tags else "<e/>" # print self.tag_to_index_map[edit_tag] adjustsoftmax = np.concatenate( (softmax, softmax[:, self.tag_to_index_map[edit_tag]].reshape( softmax.shape[0], 1)), 1) else: adjustsoftmax = softmax last_n_timings = None if ((not self.args.use_timing_data) or not timing) \ else get_last_n_features("timings", self.word_graph, len(self.word_graph)-1, n=3) new_tags = self.decoder.viterbi_incremental( adjustsoftmax, a_range=(len(adjustsoftmax) - 1, len(adjustsoftmax)), changed_suffix_only=True, timing_data=last_n_timings, words=[word]) # print "new tags", new_tags prev_output_tags = deepcopy(self.output_tags) self.output_tags = self.output_tags[:len(self.output_tags) - (len(new_tags) - 1)] + new_tags # 4. convert to standardized output format if "simple" in self.args.tags: for p in range( len(self.output_tags) - (len(new_tags) + 1), len(self.output_tags)): rps = self.output_tags[p] self.output_tags[p] = rps.replace('rm-0', 'rps id="{}"'.format(p)) if "<i" in self.output_tags[p]: self.output_tags[p] = self.output_tags[p].\ replace("<e/>", "").replace("<i", "<e/><i") else: # new_words = [word] words = get_last_n_features("words", self.word_graph, len(self.word_graph) - 1, n=len(self.word_graph) - (self.window_size - 1)) self.output_tags = convert_from_inc_disfluency_tags_to_eval_tags( self.output_tags, words, start=len(self.output_tags) - (len(new_tags)), representation=self.args.tags) if diff_only: for i, old_new in enumerate(zip(prev_output_tags, self.output_tags)): old, new = old_new if old != new: return self.output_tags[i:] return self.output_tags[len(prev_output_tags):] return self.output_tags def tag_utterance(self, utterance): """Tags entire utterance, only possible on models trained on unsegmented data. """ if not self.args.utts_presegmented: raise NotImplementedError("Tagger trained on unsegmented data,\ please call tag_prefix(words) instead.") # non segmenting self.reset() # always starts in initial state if not self.args.pos: # no pos tag model utterance = [(w, None, t) for w, p, t in utterance] # print "Warning: not using pos tags as not pos tag model" if not self.args.use_timing_data: utterance = [(w, p, None) for w, p, t in utterance] # print "Warning: not using timing durations as no timing model" for w, p, t in utterance: if self.args.pos: self.tag_new_word(w, pos=p, timing=t) return self.output_tags def rollback(self, backwards): super(DeepDisfluencyTagger, self).rollback(backwards) self.state_history = self.state_history[:len(self.state_history) - backwards] self.softmax_history = self.softmax_history[:len(self.softmax_history ) - backwards] self.decoder.rollback(backwards) def init_deep_model_internal_state(self): if self.model_type == "lstm": self.model.load_weights(c0=self.initial_c0_state, h0=self.initial_h0_state) elif self.model_type == "elman": self.model.load_weights(h0=self.initial_h0_state) def reset(self): super(DeepDisfluencyTagger, self).reset() self.word_graph = [("<s>", "<s>", 0)] * \ (self.window_size - 1) self.state_history = [] self.softmax_history = [] self.decoder.viterbi_init() self.init_deep_model_internal_state() def evaluate_fast_from_matrices(self, validation_matrices, tag_file, idx_to_label_dict): output = [] true_y = [] for v in validation_matrices: words_idx, pos_idx, extra, y, indices = v if extra: output.extend( self.model.classify_by_index(words_idx, indices, pos_idx, extra)) else: output.extend( self.model.classify_by_index(words_idx, indices, pos_idx)) true_y.extend(y) p_r_f_tags = precision_recall_fscore_support(true_y, output, average='macro') tag_summary = classification_report( true_y, output, labels=[i for i in xrange(len(idx_to_label_dict.items()))], target_names=[ idx_to_label_dict[i] for i in xrange(len(idx_to_label_dict.items())) ]) print tag_summary results = { "f1_rmtto": p_r_f_tags[2], "f1_rm": p_r_f_tags[2], "f1_tto1": p_r_f_tags[2], "f1_tto2": p_r_f_tags[2] } results.update({'f1_tags': p_r_f_tags[2], 'tag_summary': tag_summary}) return results def train_net(self, train_dialogues_filepath=None, validation_dialogues_filepath=None, model_dir=None, tag_accuracy_file_path=None): """Train the internal deep learning model from a list of dialogue matrices. """ tag_accuracy_file = open(tag_accuracy_file_path, "a") print "Verifying files..." for filepath in [ train_dialogues_filepath, validation_dialogues_filepath ]: if not verify_dialogue_data_matrices_from_folder( filepath, word_dict=self.word_to_index_map, pos_dict=self.pos_to_index_map, tag_dict=self.tag_to_index_map, n_lm=self.args.n_language_model_features, n_acoustic=self.args.n_acoustic_features): raise Exception("Dialogue vectors in wrong format!\ See README.md.") lr = self.args.lr # even if decay, start with specific lr n_extra = self.args.n_language_model_features + \ self.args.n_acoustic_features # validation matrices filepath much smaller so can store these # and preprocess them all: validation_matrices = [ np.load(validation_dialogues_filepath + "/" + fp) for fp in os.listdir(validation_dialogues_filepath) ] validation_matrices = [ dialogue_data_and_indices_from_matrix( d_matrix, n_extra, pre_seg=self.args.utts_presegmented, window_size=self.window_size, bs=self.args.bs, tag_rep=self.args.tags, tag_to_idx_map=self.tag_to_index_map, in_utterances=self.args.utts_presegmented) for d_matrix in validation_matrices ] idx_2_label_dict = {v: k for k, v in self.tag_to_index_map.items()} if not os.path.exists(model_dir): os.mkdir(model_dir) start = 1 # by default start from the first epoch best_score = 0 best_epoch = 0 print "Net training started..." for e in range(start, self.args.n_epochs + 1): tic = time.time() epoch_folder = model_dir + "/epoch_{}".format(e) if not os.path.exists(epoch_folder): os.mkdir(epoch_folder) train_loss = 0 # TODO IO is slow, where the memory allows do in one load_separately = True test = False if load_separately: for i, dialogue_f in enumerate( os.listdir(train_dialogues_filepath)): if test and i > 3: break print dialogue_f d_matrix = np.load(train_dialogues_filepath + "/" + dialogue_f) word_idx, pos_idx, extra, y, indices = \ dialogue_data_and_indices_from_matrix( d_matrix, n_extra, window_size=self.window_size, bs=self.args.bs, pre_seg=self.args.utts_presegmented ) # for i in range(len(indices)): # print i, word_idx[i], pos_idx[i], \ # y[i], indices[i] train_loss += self.model.fit(word_idx, y, lr, indices, pos_idx=pos_idx, extra_features=extra) print '[learning] file %i >>' % (i+1),\ 'completed in %.2f (sec) <<\r' % (time.time() - tic) # save the initial states we've learned to override the random self.initial_h0_state = self.model.h0.get_value() if self.args.model_type == "lstm": self.initial_c0_state = self.model.c0.get_value() # reset and evaluate simply self.reset() results = self.evaluate_fast_from_matrices( validation_matrices, tag_accuracy_file, idx_to_label_dict=idx_2_label_dict) val_score = results['f1_tags'] #TODO get best score type print "epoch training loss", train_loss print '[learning] epoch %i >>' % (e),\ 'completed in %.2f (sec) <<\r' % (time.time() - tic) print "validation score", val_score tag_accuracy_file.write( str(e) + "\n" + results['tag_summary'] + "\n%%%%%%%%%%\n") tag_accuracy_file.flush() print "saving model..." self.model.save(epoch_folder) # Epoch file dump # checking patience and decay, if applicable # stopping criterion if val_score > best_score: self.model.save(model_dir) best_score = val_score print 'NEW BEST raw labels at epoch ', e, 'best valid',\ best_score best_epoch = e # stopping criteria = if no improvement in 10 epochs if e - best_epoch >= 10: print "stopping, no improvement in 10 epochs" break if self.args.decay and (e - best_epoch) > 1: # just a steady decay if things aren't improving for 2 epochs # a hidden hyperparameter decay_rate = 0.85 lr *= decay_rate print "learning rate decayed, now ", lr if lr < 1e-5: print "stopping, below learning rate threshold" break print '[learning and testing] epoch %i >>' % (e),\ 'completed in %.2f (sec) <<\r' % (time.time()-tic) print 'BEST RESULT: epoch', best_epoch, 'valid score', best_score tag_accuracy_file.close() return best_epoch def incremental_output_from_file(self, source_file_path, target_file_path=None, is_asr_results_file=False): """Return the incremental output in an increco style given the incoming words + POS. E.g.: Speaker: KB3_1 Time: 1.50 KB3_1:1 0.00 1.12 $unc$yes NNP <f/><tc/> Time: 2.10 KB3_1:1 0.00 1.12 $unc$yes NNP <rms id="1"/><tc/> KB3_1:2 1.12 2.00 because IN <rps id="1"/><cc/> Time: 2.5 KB3_1:2 1.12 2.00 because IN <rps id="1"/><rpndel id="1"/><cc/> from an ASR increco style input without the POStags: or a normal style disfluency dectection ground truth corpus: Speaker: KB3_1 KB3_1:1 0.00 1.12 $unc$yes NNP <rms id="1"/><tc/> KB3_1:2 1.12 2.00 $because IN <rps id="1"/><cc/> KB3_1:3 2.00 3.00 because IN <f/><cc/> KB3_1:4 3.00 4.00 theres EXVBZ <f/><cc/> KB3_1:6 4.00 5.00 a DT <f/><cc/> KB3_1:7 6.00 7.10 pause NN <f/><cc/> :param source_file_path: str, file path to the input file :param target_file_path: str, file path to output in the above format :param is_asr_results_file: bool, whether the input is increco style """ if target_file_path: target_file = open(target_file_path, "w") if not self.args.do_utt_segmentation: print "not doing utt seg, using pre-segmented file" if is_asr_results_file: return NotImplementedError if 'timings' in source_file_path: print "input file has timings" if not is_asr_results_file: dialogues = [] IDs, timings, words, pos_tags, labels = \ get_tag_data_from_corpus_file(source_file_path) for dialogue, a, b, c, d in zip(IDs, timings, words, pos_tags, labels): dialogues.append((dialogue, (a, b, c, d))) else: print "no timings in input file, creating fake timings" raise NotImplementedError for speaker, speaker_data in dialogues: # if "4565" in speaker: quit() print speaker self.reset() # reset at the beginning of each dialogue if target_file_path: target_file.write("Speaker: " + str(speaker) + "\n\n") timing_data, lex_data, pos_data, labels = speaker_data # iterate through the utterances # utt_idx = -1 current_time = 0 for i in range(0, len(timing_data)): # print i, timing_data[i] _, end = timing_data[i] if (not self.args.do_utt_segmentation) \ and "<t" in labels[i]: self.reset() # reset after each utt if non pre-seg # utt_idx = frames[i] timing = None if 'timings' in source_file_path and self.args.use_timing_data: timing = end - current_time word = lex_data[i] pos = pos_data[i] diff = self.tag_new_word(word, pos, timing, diff_only=True, rollback=0) current_time = end if target_file_path: target_file.write("Time: " + str(current_time) + "\n") new_words = lex_data[i - (len(diff) - 1):i + 1] new_pos = pos_data[i - (len(diff) - 1):i + 1] new_timings = timing_data[i - (len(diff) - 1):i + 1] for t, w, p, tag in zip(new_timings, new_words, new_pos, diff): target_file.write("\t".join( [str(t[0]), str(t[1]), w, p, tag])) target_file.write("\n") target_file.write("\n") target_file.write("\n") def train_decoder(self, tag_file): raise NotImplementedError def save_decoder_model(self, dir_path): raise NotImplementedError
class SimpleSLU: def __init__(self): self.__semantic_instance_list = [] self.__speech_act_instance_list = [] self.__semantic_model = None self.__speech_act_model = None self.__speech_act_lb = None def load_model(self, modelfile): with open('%s.act.model' % modelfile, 'r') as f: self.__speech_act_model, self.__speech_act_lb = pickle.load(f) self.__semantic_model = CRFTagger(verbose=True) self.__semantic_model.set_model_file('%s.semantic.model' % modelfile) return True def add_instance(self, utter, speech_act, semantic_tagged): tokenized = self.__tokenize(utter, semantic_tagged) if tokenized is None: return False semantic_instance = [] for word, (bio, tag, attrs) in tokenized: if bio is None: sem_label = 'O' else: cat = None for attr, val in attrs: if attr == 'cat': cat = val sem_label = '%s-%s_%s' % (bio, tag, cat) semantic_instance.append((unicode(word.lower()), unicode(sem_label))) self.__semantic_instance_list.append(semantic_instance) sa_label_list = [] for sa in speech_act: sa_labels = ['%s_%s' % (sa['act'], attr) for attr in sa['attributes']] sa_label_list += sa_labels sa_label_list = sorted(set(sa_label_list)) word_feats = ' '.join([word.lower() for word, _ in tokenized]) self.__speech_act_instance_list.append((word_feats, sa_label_list)) return True def train(self, modelfile): sa_feats = [x for x, _ in self.__speech_act_instance_list] sa_labels = [y for _, y in self.__speech_act_instance_list] self.__speech_act_lb = preprocessing.MultiLabelBinarizer() sa_labels = self.__speech_act_lb.fit_transform(sa_labels) self.__speech_act_model = Pipeline([ ('vectorizer', CountVectorizer()), ('tfidf', TfidfTransformer()), ('clf', OneVsRestClassifier(LinearSVC(verbose=True)))]) self.__speech_act_model.fit(sa_feats, sa_labels) with open('%s.act.model' % modelfile, 'wb') as f: pickle.dump((self.__speech_act_model, self.__speech_act_lb), f) self.__semantic_model = CRFTagger(verbose=True) self.__semantic_model.train(self.__semantic_instance_list, '%s.semantic.model' % modelfile) def pred(self, utter): tokenized = self.__tokenize(utter) word_feats = ' '.join([word.lower() for word, _ in tokenized]) pred_act = self.__speech_act_lb.inverse_transform(self.__speech_act_model.predict([word_feats])) pred_semantic = self.__semantic_model.tag([word.lower() for word, _ in tokenized]) return (pred_act, pred_semantic) def __tokenize(self, utter, semantic_tagged=None): result = None if semantic_tagged is None: result = [(word, None) for word in nltk.word_tokenize(utter)] else: parser_raw = SemanticTagParser(False) parser_tagged = SemanticTagParser(False) segmented = ' '.join(nltk.word_tokenize(utter)) tagged = ' '.join(semantic_tagged) parser_raw.feed(segmented) parser_tagged.feed(tagged) raw_chr_seq = parser_raw.get_chr_seq() raw_space_seq = parser_raw.get_chr_space_seq() tagged_chr_seq = parser_tagged.get_chr_seq() tagged_space_seq = parser_tagged.get_chr_space_seq() if raw_chr_seq == tagged_chr_seq: merged_space_seq = [ x or y for x, y in zip(raw_space_seq, tagged_space_seq)] word_seq = parser_tagged.tokenize(merged_space_seq) tag_seq = parser_tagged.get_word_tag_seq() result = [(word, tag) for word, tag in zip(word_seq, tag_seq)] return result
# -*- coding: utf-8 -*- from nltk import word_tokenize from nltk.tokenize import RegexpTokenizer from nltk.tag import CRFTagger ct = CRFTagger() ct.set_model_file('./input/tagger/indonesian_tagger') # memasukkan fitur ke dictionary def insert_dict(kata, fitur): if kata not in fitur: fitur[kata] = 1 else: fitur[kata] += 1 return fitur # berikan tag pada def beri_tag(kata): kata_tag = ct.tag_sents([word_tokenize(kata)]) return kata_tag def tf_baseline(kalimat): fitur = {} clean_punct = RegexpTokenizer(r'\w+') arr = [clean_punct.tokenize(kalimat.lower())] hasil = ct.tag_sents(arr) for sentence in hasil: for word_tag in sentence: