def segment(labels, reviews): segmented = [] print('Creating BOW') # seg = StanfordSegmenter('../../datasets/data-hauyi/stanford-segmenter-2018-10-16') os.environ[ "STANFORD_SEGMENTER"] = '../datasets/data-hauyi/stanford-segmenter-2018-10-16' seg = StanfordSegmenter( '../datasets/data-hauyi/stanford-segmenter-2018-10-16/stanford-segmenter-3.9.2.jar' ) seg.default_config('zh', ) count = 0 file_out = open('reviews.txt', 'a+') for i in range(len(reviews)): # print(i) s = seg.segment(reviews[i]) l = labels[i] # print(s) line = str(l) + ' ' + s file_out.write(line) segmented.append(s) # print('Tokenize: ') # print(seg.tokenize(s)) count = count + 1 # if count > 5: # break print('Count: ', count) return (segmented)
def segmenter(sentence): r""" Stanford Word Segmenter for Chinese. Split Chinese sentence into a sequence of words. Args: sentence:A Chinese sentence Returns: A list decode in utf-8 Example: sentence="广东外语外贸大学是一所具有鲜明国际化特色的广东省属重点大学,是华南地区国际化人才培养和外国语言文化、对外经济贸易、国际战略研究的重要基地。" [u'\u5e7f\u4e1c', u'\u5916\u8bed', u'\u5916\u8d38', u'\u5927\u5b66', u'\u662f', u'\u4e00', u'\u6240', u'\u5177\u6709', u'\u9c9c\u660e', u'\u56fd\u9645\u5316', u'\u7279\u8272', u'\u7684', u'\u5e7f\u4e1c', u'\u7701\u5c5e', u'\u91cd\u70b9', u'\u5927\u5b66', u'\uff0c', u'\u662f', u'\u534e\u5357', u'\u5730\u533a', u'\u56fd\u9645\u5316', u'\u4eba\u624d', u'\u57f9\u517b', u'\u548c', u'\u5916\u56fd', u'\u8bed\u8a00', u'\u6587\u5316', u'\u3001', u'\u5bf9\u5916', u'\u7ecf\u6d4e', u'\u8d38\u6613', u'\u3001', u'\u56fd\u9645', u'\u6218\u7565', u'\u7814\u7a76', u'\u7684', u'\u91cd\u8981', u'\u57fa\u5730', u'\u3002'] """ from nltk.tokenize.stanford_segmenter import StanfordSegmenter #初始化斯坦福中文分词器 segmenter = StanfordSegmenter( path_to_jar= 'D:/python/nltk-3.1/nltk/chin/stanford-segmenter-2014-08-27/stanford-segmenter-3.4.1.jar', path_to_sihan_corpora_dict= 'D:/python/nltk-3.1/nltk/chin/stanford-segmenter-2014-08-27/data', path_to_model= 'D:/python/nltk-3.1/nltk/chin./stanford-segmenter-2014-08-27/data/pku.gz', path_to_dict= 'D:/python/nltk-3.1/nltk/chin./stanford-segmenter-2014-08-27/data/dict-chris6.ser.gz' ) #加载中文分词模型 sent = segmenter.segment(sentence) #分词 return sent.split()
def splitAllWord(typeOfDataset="dev"): from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter() segmenter.default_config('zh') maxCount = 2000000 pathOfDev = "dataset/task1/%s.tsv" % typeOfDataset dfOfDev = pd.read_csv(pathOfDev, delimiter="\t") pathOfNewDev = "%s_split.tsv" % typeOfDataset count = 0 with open(pathOfNewDev, "w", encoding='utf-8') as fw: for row in dfOfDev.iterrows(): if count >= maxCount: break if count % 100 == 0: print("[%s]count = %s" % (typeOfDataset, count)) label = row[1]['label'] fw.write(str(label)) fw.write("\t") sentence = row[1]['text_a'] segmentOfSentence = segmenter.segment(sentence) for word in segmentOfSentence.split(): fw.write(word) fw.write(" ") fw.write("\n") count += 1
def ch_standseg(mystr): segmenter = StanfordSegmenter( path_to_jar=r"E:\tools\stanfordNLTK\jar\stanford-segmenter.jar", path_to_slf4j=r"E:\tools\stanfordNLTK\jar\slf4j-api.jar", path_to_sihan_corpora_dict=r"E:\tools\stanfordNLTK\jar\data", path_to_model=r"E:\tools\stanfordNLTK\jar\data\pku.gz", path_to_dict=r"E:\tools\stanfordNLTK\jar\data\dict-chris6.ser.gz") result = segmenter.segment(mystr) print(result)
class TMStanfordTokenizer(): models = {'ZH': 'ctb.gz', 'AR': 'arabic-segmenter-atb+bn+arztrain.ser.gz'} dics = {'ZH': 'dict-chris6.ser.gz', 'AR': ''} def __init__(self, language): self.language = language model = self.models.get(language) dic = self.dics.get(language) if not model: raise (Exception( "Unsupported language for tokenizer: {}".format(language))) # Initialize Stanford Tokenizer self.tm_tokenize = StanfordSegmenter( path_to_jar=os.path.join(stanford_tokenizer_home, 'stanford-segmenter-3.6.0.jar'), path_to_model=os.path.join(stanford_tokenizer_home, 'data', model), path_to_dict=os.path.join(stanford_tokenizer_home, 'data', dic), path_to_sihan_corpora_dict=os.path.join(stanford_tokenizer_home, 'data'), path_to_slf4j=os.path.join(stanford_tokenizer_home, 'slf4j-api.jar')) #Input: String #Output: 这 是 斯坦福 中文 分词 器 测试 def process(self, sentences): text = self.tm_tokenize.segment(sentences).strip('\n') if re.search(TOK_PATTERN, text): # Check if the text have tags text = XmlUtils.join_tags(text, JOIN_PATTERN) return text def tokenize_sent(self, text): if self.language == 'ZH': return [s + '。' for s in text.split('。') if s] # Split by sentence chinese #self.tm_tokenize.segment_sents(text) return [text]
#coding:utf-8 from nltk.tokenize.stanford_segmenter import StanfordSegmenter #中文分词 segmenter=StanfordSegmenter( path_to_jar="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/stanford-segmenter-3.5.2.jar", path_to_slf4j="/home/hsiao/Develops/nlp/stanford-corenlp-full-2016-10-31/slf4j-api.jar", path_to_sihan_corpora_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data", path_to_model="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/pku.gz", path_to_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/dict-chris6.ser.gz" ) str="我在我在博客园开了一个博客。" print (segmenter.segment(str)) #英文分词 from nltk.tokenize import StanfordTokenizer tokenizer=StanfordTokenizer(path_to_jar=r"/home/hsiao/Develops/nlp/stanford-parser-full-2016-10-31/stanford-parser.jar") sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." print (tokenizer.tokenize(sent)) #中文命名实体识别 from nltk.tag import StanfordNERTagger chi_tagger=StanfordNERTagger(model_filename=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/classifiers/chinese.misc.distsim.crf.ser.gz' ,path_to_jar=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/stanford-ner.jar') print (chi_tagger.tag('四川省 成都 信息 工程 大学 我 在 博客 园 开 了 一个 博客 , 我 的 博客 名叫 伏 草 惟 存 , 写 了 一些 自然语言 处理 的 文章 。\r\n'.split()))
class NLPCore: """ nlp processing including Stanford Word Segmenter, Stanford POS Tagger, Stanford Named Entity Recognizer and Stanford Parser """ def __init__(self): self.root_path = '../Models/stanfordNLP/' # word segmenter self.segmenter = StanfordSegmenter( path_to_jar=self.root_path + "stanford-segmenter.jar", path_to_slf4j=self.root_path + "log4j-over-slf4j.jar", path_to_sihan_corpora_dict=self.root_path + "segmenter/", path_to_model=self.root_path + "segmenter/pku.gz", path_to_dict=self.root_path + "segmenter/dict-chris6.ser.gz") # pos tagger self.posTagger = StanfordPOSTagger( self.root_path + 'pos-tagger/chinese-distsim.tagger', path_to_jar=self.root_path + "stanford-postagger.jar") # named entity recognizer self.nerTagger = StanfordNERTagger( self.root_path + 'ner/chinese.misc.distsim.crf.ser.gz', path_to_jar=self.root_path + 'stanford-ner.jar') self.parser = StanfordDependencyParser( model_path=self.root_path + 'lexparser/chinesePCFG.ser.gz', path_to_jar=self.root_path + 'stanford-parser.jar', path_to_models_jar=self.root_path + 'stanford-parser-3.7.0-models.jar', encoding='gbk') def split_sent_stanford(self, textPair): """ Stanford Word Segmenter, input should be raw text :return: also TextPair with raw string of results """ t1 = self.segmenter.segment(textPair.t1) t2 = self.segmenter.segment(textPair.t1) if DEBUG: print(t1, t2) return text_pair.TextPair(t1, t2, textPair.label) def split_sents_stanford(self, textPairs): """ Stanford Word Segmenter, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1 for textPair in textPairs] sents2 = [textPair.t2 for textPair in textPairs] split1 = self.segmenter.segment_sents(sents1).split('\n') split2 = self.segmenter.segment_sents(sents2).split('\n') rlist = [] for i in range(len(textPairs)): rlist.append( text_pair.TextPair(split1[i], split2[i], textPairs[i].label)) if DEBUG: print(split1[i], split2[i]) return rlist def split_sent_jieba(self, textPair): jieba.setLogLevel('INFO') ger1 = jieba.cut(textPair.t1) ger2 = jieba.cut(textPair.t2) t1 = ' '.join(ger1) t2 = ' '.join(ger2) return text_pair.TextPair(t1, t2, textPair.label) def pos_tag(self, textPair): """ Stanford POS Tagger, input should be splitted :return: also TextPair with raw string of results """ t1_s = textPair.t1.split() t2_s = textPair.t2.split() t1_tag = ' '.join([ele[1] for ele in self.posTagger.tag(t1_s)]) t2_tag = ' '.join([ele[1] for ele in self.posTagger.tag(t2_s)]) if DEBUG: print(t1_tag, t2_tag) return text_pair.TextPair(t1_tag, t2_tag, textPair.label) def pos_tag_pairs(self, textPairs): """ Stanford POS Tagger, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1.split() for textPair in textPairs] sents2 = [textPair.t2.split() for textPair in textPairs] tag1 = self.posTagger.tag_sents(sents1) tag2 = self.posTagger.tag_sents(sents2) rlist = [] for i in range(len(tag1)): t1_tag = ' '.join([ele[1] for ele in tag1[i]]) t2_tag = ' '.join([ele[1] for ele in tag2[i]]) rlist.append(text_pair.TextPair(t1_tag, t2_tag, textPairs[i].label)) if DEBUG: print(t1_tag, t2_tag) return rlist def ner_tag(self, textPair): """ Stanford Named Entity Recognizer, input should be splitted :return: also TextPair with raw string of results """ t1_s = textPair.t1.split() t2_s = textPair.t2.split() t1_ner = ' '.join( [ele[0] + '#' + ele[1] for ele in self.nerTagger.tag(t1_s)]) t2_ner = ' '.join( [ele[0] + '#' + ele[1] for ele in self.nerTagger.tag(t2_s)]) if DEBUG: print(t1_ner, t2_ner) return text_pair.TextPair(t1_ner, t2_ner, textPair.label) def ner_tag_pairs(self, textPairs): """ Stanford Named Entity Recognizer, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1.split() for textPair in textPairs] sents2 = [textPair.t2.split() for textPair in textPairs] tag1 = self.nerTagger.tag_sents(sents1) tag2 = self.nerTagger.tag_sents(sents2) rlist = [] for i in range(len(tag1)): t1_ner = ' '.join([ele[0] + '#' + ele[1] for ele in tag1[i]]) t2_ner = ' '.join([ele[0] + '#' + ele[1] for ele in tag2[i]]) rlist.append(text_pair.TextPair(t1_ner, t2_ner, textPairs[i].label)) if DEBUG: print(t1_ner, t2_ner) return rlist def depen_parse(self, textPair): """ Stanford Dependency Parser, input should be splitted :return: also TextPair with raw string of results """ print([p.tree() for p in self.parser.raw_parse(textPair.t1)])
# coding: utf-8 from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tag import StanfordPOSTagger segmenter = StanfordSegmenter( java_class='edu.stanford.nlp.ie.crf.CRFClassifier', path_to_slf4j="/home/lcy/stanford-segmenter/slf4j-api.jar", path_to_jar="/home/lcy/stanford-segmenter/stanford-segmenter.jar", path_to_sihan_corpora_dict="/home/lcy/stanford-segmenter/data/", path_to_model="/home/lcy/stanford-segmenter/data/pku.gz", path_to_dict="/home/lcy/stanford-segmenter/data/dict-chris6.ser.gz" ) f = open("wiki.zh.text", "r") # 读取文本 string = f.read().decode("utf-8") result = segmenter.segment(string) # 分词 # print result f = open("fenci.txt", "w") # 将结果保存到另一个文档中 f.write(result.encode("utf-8")) f.close()
str_de1 = 'Jenes Mädchen ist sehr hübsch.' snt_de = "Sie schaut mich an, als ob sie mich nicht verstände." str_en1 = 'In other words, counterfactual thinking influences how satisfied each athlete feels.' snt = "Bills on ports and immigration were submitted by Senator Brownback, Republican of Kansas" snt_en2 = 'I like cat and dog' cldg = dep_parser_en.raw_parse(snt_en2) dep = next(cldg) print(dep.to_conll(10)) cldg = dep_parser_en.collapsed_parse(snt_en2) dep = next(cldg) print(dep.to_conll(10)) cldg = dep_parser_en.collapsed_parse(snt) dep = next(cldg) print(dep.to_conll(10)) ch_seg_snt = segmenter.segment(snt_ch).strip('\n').strip('。') cldg = dep_parser_cn.collapsed_parse(ch_seg_snt) dep = next(cldg) print(dep.to_conll(10)) cldg = dep_parser_de.raw_parse(snt_de) dep = next(cldg) print(dep.to_conll(10)) # http://stackoverflow.com/questions/31975893/how-can-i-get-a-grammaticalstructure-object-for-a-german-sentence-using-the-stan cldg = get_dep_str(snt_de, lan='de', de_parser='parzu') print(cldg) #dep_str = get_dep_str(str_ch1, lan='ch', ch_parser='hit') #print(dep_str)
# find_entity_t = test.find_entity() # find_VP_t = test.firstVP() # test.drawTree() test.show(firstNP_t) # test.show(find_entity_t) # test.show(find_VP_t) # # test.show(find_entity_t) # test.show(firstMinNP_t) result = test.find_realtionship(firstNP_t) print(result) test.drawTree() # # # print(test.rel) # test.show(test.find_realtionship()) # 对比实验 chi_parser = StanfordParser(path_to_jar='../stanford-parser-full-2018-02-27/stanford-parser.jar', path_to_models_jar='../stanford-parser-full-2018-02-27/stanford-parser-3.9.1-models.jar', model_path='../stanford-parser-full-2018-02-27/stanford-parser-3.9.1-models/edu/stanford/nlp/models/lexparser/chinesePCFG.ser.gz') data_dir='../stanford-segmenter-2018-02-27/' segmenter = StanfordSegmenter(path_to_jar=data_dir+"stanford-segmenter-3.9.1.jar", path_to_sihan_corpora_dict=data_dir+"/data", path_to_model=data_dir+"/data/pku.gz", path_to_dict=data_dir+"/data/dict-chris6.ser.gz", java_class='edu.stanford.nlp.ie.crf.CRFClassifier', ) result=segmenter.segment(test_str) result_ls = result.split() ch_tree = list(chi_parser.parse(result_ls))[0] ch_tree.draw() # print(result)
elif (tokeniser == 'stanfordpku'): from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tokenize import StanfordTokenizer workingdirectory = os.getcwd() segmenter = StanfordSegmenter( path_to_jar=os.path.join(workingdirectory, 'stanford-segmenter.jar'), path_to_slf4j=os.path.join(workingdirectory, 'slf4j-api.jar'), path_to_sihan_corpora_dict=os.path.join(workingdirectory, 'data'), path_to_model=os.path.join(workingdirectory, 'data', 'pku.gz'), path_to_dict=os.path.join(workingdirectory, 'data', 'dict-chris6.ser.gz')) tokenizer = StanfordTokenizer( path_to_jar=os.path.join(workingdirectory, 'stanford-parser.jar')) n = 1 for line in open(sourcefile): token = segmenter.segment(line) words = tokenizer.tokenize(token) with open('%s%s.txt' % (prefix, n), "w", encoding='utf-8') as resultfile: resultwrite = csv.writer(resultfile) for word in words: resultwrite.writerow([word]) n = n + 1 print('Done') elif (tokeniser == 'stanfordctb'): from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tokenize import StanfordTokenizer workingdirectory = os.getcwd() segmenter = StanfordSegmenter( path_to_jar=os.path.join(workingdirectory, 'stanford-segmenter.jar'), path_to_slf4j=os.path.join(workingdirectory, 'slf4j-api.jar'),
############################### ###tokenize chinese#### print '########Starting tokenization###########\n' ## from nltk.tokenize.stanford_segmenter import StanfordSegmenter pre_path = '/home/db32555/MM/stanford-segmenter/' segmenter = StanfordSegmenter(path_to_jar=pre_path+'stanford-segmenter-3.4.1.jar', path_to_sihan_corpora_dict=pre_path+'./data', path_to_model=pre_path+'./data/pku.gz', path_to_dict=pre_path+'./data/dict-chris6.ser.gz') ##setup_end## from nltk import word_tokenize ##setup_end_eng## for node in corpus: index = corpus.index(node) chinese = node[1] chinese = unicode(chinese, 'utf-8') tmp_segmented = segmenter.segment(chinese) tmp_segmented = tmp_segmented.split(" ") # del corpus[index][1] corpus[index].append(tmp_segmented) print tmp_segmented ##this is chinese english = node[0] english = unicode(english, 'utf-8') english = word_tokenize(english) del corpus[index][0] corpus[index].append(english) print english ##this is english print '########End of tokenization###########\n'
num2word = [] word_dict = {} lyrics = pd.read_csv('./data/%s.csv' % (lyric_document), encoding='utf-8', index_col='title') # song_num = len(lyrics['lyric']) # 统计歌曲数目 title_set = lyrics.index[0] # 初始化歌名集 和 歌词集 song_set = lyrics['lyric'][0] for i in range(song_num): if i != 0: title_set = title_set + u'\n' + lyrics.index[i] song_set = song_set + u'\n' + lyrics['lyric'][i] title_result = segmenter.segment(title_set) song_result = segmenter.segment(song_set) sentences = [] count_num = 0 #将每句话放入sentences[]中 temp = '' for i in song_result: if i != u'\n': temp += i else: temp += i sentences.append(temp) temp = '' count_num += 1
# cols = ["用户ID","用户问题"] # df = pd.read_excel(filename) # df_selected = df[cols] # print(df[cols].head(2)) # df_segment = segmenter.segment(df_selected["用户问题"].head(1)) # print(df_segment) import xlrd xls_data = xlrd.open_workbook(filename) table = xls_data.sheets()[0] nrows = table.nrows # 行数 ncols = table.ncols # 列数 list2 = [] for rownum in range(1, 100): row = table.row_values(rownum) if row: list2.append(segmenter.segment(row[2]).split(" ")) print(list2) # In[5]: # In[1]: import nltk text = nltk.word_tokenize("And now for something compeletely differently") print(nltk.pos_tag(text)) # In[ ]:
# -*- coding: utf-8 -*- from __future__ import unicode_literals from nltk.tokenize.stanford_segmenter import StanfordSegmenter seg = StanfordSegmenter( path_to_jar=r'D:\temp\stanford\stanford-segmenter.jar', path_to_slf4j=r'D:\temp\stanford\slf4j-api.jar', path_to_sihan_corpora_dict=r'D:\temp\stanford\data', path_to_model=r'D:\temp\stanford\data\ctb.gz', path_to_dict=r'D:\temp\stanford\data\dict-chris6.ser.gz', ) # sentence = "这是斯坦福中文分词器测试" sentence = "操你大爷,狗日的" res = seg.segment(sentence) print res # import jieba # ss = jieba.cut(sentence) # print ' '.join(list(ss)), type(ss)
# # u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n' # segmenter.segment_file("test.simp.utf8") # # u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ... ##下载 # import nltk # nltk.download() from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter( path_to_jar="stanford-segmenter-3.6.0.jar", path_to_slf4j = "slf4j-api.jar", path_to_sihan_corpora_dict="./data", path_to_model="./data/pku.gz", path_to_dict="./data/dict-chris6.ser.gz") sentence = u"这是斯坦福中文分词器测试" segmenter.segment(sentence) # >>> u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n' segmenter.segment_file("test.simp.utf8") # >>> u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ... # 英文测试 # import nltk # text = 'i am a good boy.you are a bad girl' # sens = nltk.sent_tokenize(text) # print(sens) # words = [] # for sent in sens: # words.append(nltk.word_tokenize(sent)) # for line in words: # print(line) #
from nltk.tokenize.stanford_segmenter import StanfordSegmenter seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar') seg = StanfordSegmenter() seg.default_config('zh') sent = u'这是斯坦福中文分词器测试' print(seg.segment(sent))
# encoding: utf-8 import nltk from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tag import StanfordNERTagger segmenter = StanfordSegmenter( #分词依赖的jar包 path_to_jar= r"/home/jiangix/document/stanford-segmenter/stanford-segmenter.jar", path_to_slf4j=r"/home/jiangix/document/slf4j/slf4j-api.jar", #分词数据文件夹 path_to_sihan_corpora_dict= r"/home/jiangix/document/stanford-segmenter/data", #基于北大在2005backoof上提供的人名日报语料库 path_to_model=r"/home/jiangix/document/stanford-segmenter/data/pku.gz", path_to_dict= r"/home/jiangix/document/stanford-segmenter/data/dict-chris6.ser.gz") segmenter.default_config('zh') result = segmenter.segment(u'我喜欢学习编程') chi_tagger = StanfordNERTagger( model_filename= r"/home/jiangix/document/stanford-chinese-corenlp-models/chinese.misc.distsim.crf.ser.gz", path_to_jar=r"/home/jiangix/document/stanford-ner/stanford-ner.jar") for word, tag in chi_tagger.tag(result.split()): print(word, tag)
sumContain = '' for f in fileNames: try: if data_type == 'CIRB010': root = ET.parse(dirPath+'/'+f).getroot() date = root[0][1].text.strip() title = root[0][2].text.strip() text = '' for p in root[0][3]: text += p.text.strip() contain = date + title + text sumContain += contain else: fin = open(dirPath+'/'+f, 'r') for line in fin.readlines(): sumContain += line.strip() except: a = '' build_time += 1. parsed_data = segmenter.segment(sumContain).split() for w in parsed_data: vocabSet.add(w) print >> sys.stderr, '\rdone building '+str(float("{0:.2f}".format(build_time/total*100.)))+'% vocabulary set ', print >> sys.stderr, '\nstart dumping vocabulary set' vocab.dumpVocab(outputfile, vocabSet) print >> sys.stderr, 'done dumping vocabulary set'
def segment(sentence): path = '/media/razor/Files/Python27/nltk_data/stanford-segmenter-2015-12-09/' segmenter = StanfordSegmenter(path + 'stanford-segmenter-3.6.0.jar', path + 'slf4j-api.jar', path + 'data/pku.gz', path + 'data/dict-chris6.ser.gz') return segmenter.segment(u'我爱北京天安门')
from nltk.tokenize.stanford import CoreNLPTokenizer # from nltk.tokenize.stanford import path = "D:/www/data/nlpsoftware/stanford-segmenter" segmenter = StanfordSegmenter( path_to_jar=path + "/stanford-segmenter.jar", path_to_sihan_corpora_dict=path + "/data", path_to_model=path + "/data/pku.gz", path_to_dict=path + "/data/dict-chris6.ser.gz", java_class='edu.stanford.nlp.ie.crf.CRFClassifier') # sentence = u"这是斯坦福中文分词器测试" sentence = u"工信处女干事每月经过 下属 科室都要亲口交代24口交换机等技术性器件的安装工作" segmenter.tokenize_sents(u"工信处") result = segmenter.segment(sentence) result2 = segmenter.segment_file( "D:/www/data/nlpdata/icwb2-data/testing/pku_test.utf8") clean_content = "D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content.txt" # clean_content_out="D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content_out.txt" # result3 = segmenter.segment_file(clean_content) print(type(result2)) # with open(clean_content_out,'wb+') as f: # f.writelines([(s+"\r\n").encode('utf8') for s in clean_content_out]) print(result2) # outfile = open("D:/www/data/nlpsoftware/outfile.txt",'w') # outfile.write(result) # outfile.close() # # stanford_postagger="D:\\www\\data/nlpsoftware/stanford-postagger-full-2017-06-09\\stanford-postagger.jar"
#coding:UTF-8 from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter( path_to_jar= r"D:\StanfordNLP\stanford-segmenter\stanford-segmenter-3.6.0.jar", path_to_slf4j=r"D:\StanfordNLP\stanford-segmenter\slf4j-api.jar", path_to_sihan_corpora_dict=r"D:\StanfordNLP\stanford-segmenter\data", path_to_model=r"D:\StanfordNLP\stanford-segmenter\data\pku.gz", path_to_dict=r"D:\StanfordNLP\stanford-segmenter\data\dict-chris6.ser.gz") str = u"我在博客园开了一个博客,我的博客名叫伏草惟存,写了一些自然语言处理的文章。" result = segmenter.segment(str) print result
from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter( path_to_jar= r"/Users/cln/stanford-corenlp/segmenter/stanford-segmenter-3.9.1.jar", path_to_slf4j=r"/Users/cln/stanford-corenlp/slf4j-api.jar", java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", # 分词模型 path_to_model=r"/Users/cln/stanford-corenlp/segmenter/data/pku.gz", path_to_dict= r"/Users/cln/stanford-corenlp/segmenter/data/dict-chris6.ser.gz", path_to_sihan_corpora_dict=r"/Users/cln/stanford-corenlp/segmenter/data") strs = "中国的一带一路带动了很多国家的发展。" # 汉语分词 ch_result = segmenter.segment(strs) print('汉语分词:\n', ch_result) print(type(ch_result), '\n') from nltk.tokenize import word_tokenize sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." # 英语分词 rr = word_tokenize(sent) print('英语分词:\n', rr, '\n') from nltk.tag import StanfordNERTagger # 英文命名实体识别 eng_tagger = StanfordNERTagger( r'/Users/cln/stanford-corenlp/stanford-ner/classifiers/english.all.3class.distsim.crf.ser.gz',