def segment(labels, reviews): segmented = [] print('Creating BOW') # seg = StanfordSegmenter('../../datasets/data-hauyi/stanford-segmenter-2018-10-16') os.environ[ "STANFORD_SEGMENTER"] = '../datasets/data-hauyi/stanford-segmenter-2018-10-16' seg = StanfordSegmenter( '../datasets/data-hauyi/stanford-segmenter-2018-10-16/stanford-segmenter-3.9.2.jar' ) seg.default_config('zh', ) count = 0 file_out = open('reviews.txt', 'a+') for i in range(len(reviews)): # print(i) s = seg.segment(reviews[i]) l = labels[i] # print(s) line = str(l) + ' ' + s file_out.write(line) segmented.append(s) # print('Tokenize: ') # print(seg.tokenize(s)) count = count + 1 # if count > 5: # break print('Count: ', count) return (segmented)
def segmenter(sentence): r""" Stanford Word Segmenter for Chinese. Split Chinese sentence into a sequence of words. Args: sentence:A Chinese sentence Returns: A list decode in utf-8 Example: sentence="广东外语外贸大学是一所具有鲜明国际化特色的广东省属重点大学,是华南地区国际化人才培养和外国语言文化、对外经济贸易、国际战略研究的重要基地。" [u'\u5e7f\u4e1c', u'\u5916\u8bed', u'\u5916\u8d38', u'\u5927\u5b66', u'\u662f', u'\u4e00', u'\u6240', u'\u5177\u6709', u'\u9c9c\u660e', u'\u56fd\u9645\u5316', u'\u7279\u8272', u'\u7684', u'\u5e7f\u4e1c', u'\u7701\u5c5e', u'\u91cd\u70b9', u'\u5927\u5b66', u'\uff0c', u'\u662f', u'\u534e\u5357', u'\u5730\u533a', u'\u56fd\u9645\u5316', u'\u4eba\u624d', u'\u57f9\u517b', u'\u548c', u'\u5916\u56fd', u'\u8bed\u8a00', u'\u6587\u5316', u'\u3001', u'\u5bf9\u5916', u'\u7ecf\u6d4e', u'\u8d38\u6613', u'\u3001', u'\u56fd\u9645', u'\u6218\u7565', u'\u7814\u7a76', u'\u7684', u'\u91cd\u8981', u'\u57fa\u5730', u'\u3002'] """ from nltk.tokenize.stanford_segmenter import StanfordSegmenter #初始化斯坦福中文分词器 segmenter = StanfordSegmenter( path_to_jar= 'D:/python/nltk-3.1/nltk/chin/stanford-segmenter-2014-08-27/stanford-segmenter-3.4.1.jar', path_to_sihan_corpora_dict= 'D:/python/nltk-3.1/nltk/chin/stanford-segmenter-2014-08-27/data', path_to_model= 'D:/python/nltk-3.1/nltk/chin./stanford-segmenter-2014-08-27/data/pku.gz', path_to_dict= 'D:/python/nltk-3.1/nltk/chin./stanford-segmenter-2014-08-27/data/dict-chris6.ser.gz' ) #加载中文分词模型 sent = segmenter.segment(sentence) #分词 return sent.split()
def segment_sentences(sentence_list): segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'stanford-segmenter-3.9.1.jar'), path_to_slf4j=os.path.join(stanford_corenlp_path, 'slf4j-api-1.7.25.jar'), path_to_sihan_corpora_dict=os.path.join( stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data'), path_to_model=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'pku.gz'), path_to_dict=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'dict-chris6.ser.gz'), sihan_post_processing='true') result = segmenter.segment_sents(sentence_list) result = result.strip() segmented_list = re.split(os.linesep, result) if len(segmented_list[-1]) == 0: segmented_list = segmented_list[:-1] if len(segmented_list) != len(sentence_list): for i in range(len(segmented_list)): ss = ''.join(segmented_list[i].split()) if ss != sentence_list[i]: print(i, '|', segmented_list[i], '|', sentence_list[i]) # break print(len(segmented_list), len(sentence_list)) assert len(segmented_list) == len(sentence_list) return segmented_list
def splitAllWord(typeOfDataset="dev"): from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter() segmenter.default_config('zh') maxCount = 2000000 pathOfDev = "dataset/task1/%s.tsv" % typeOfDataset dfOfDev = pd.read_csv(pathOfDev, delimiter="\t") pathOfNewDev = "%s_split.tsv" % typeOfDataset count = 0 with open(pathOfNewDev, "w", encoding='utf-8') as fw: for row in dfOfDev.iterrows(): if count >= maxCount: break if count % 100 == 0: print("[%s]count = %s" % (typeOfDataset, count)) label = row[1]['label'] fw.write(str(label)) fw.write("\t") sentence = row[1]['text_a'] segmentOfSentence = segmenter.segment(sentence) for word in segmentOfSentence.split(): fw.write(word) fw.write(" ") fw.write("\n") count += 1
def __init__(self): self.root_path = '../Models/stanfordNLP/' # word segmenter self.segmenter = StanfordSegmenter( path_to_jar=self.root_path + "stanford-segmenter.jar", path_to_slf4j=self.root_path + "log4j-over-slf4j.jar", path_to_sihan_corpora_dict=self.root_path + "segmenter/", path_to_model=self.root_path + "segmenter/pku.gz", path_to_dict=self.root_path + "segmenter/dict-chris6.ser.gz") # pos tagger self.posTagger = StanfordPOSTagger( self.root_path + 'pos-tagger/chinese-distsim.tagger', path_to_jar=self.root_path + "stanford-postagger.jar") # named entity recognizer self.nerTagger = StanfordNERTagger( self.root_path + 'ner/chinese.misc.distsim.crf.ser.gz', path_to_jar=self.root_path + 'stanford-ner.jar') self.parser = StanfordDependencyParser( model_path=self.root_path + 'lexparser/chinesePCFG.ser.gz', path_to_jar=self.root_path + 'stanford-parser.jar', path_to_models_jar=self.root_path + 'stanford-parser-3.7.0-models.jar', encoding='gbk')
def ch_standseg(mystr): segmenter = StanfordSegmenter( path_to_jar=r"E:\tools\stanfordNLTK\jar\stanford-segmenter.jar", path_to_slf4j=r"E:\tools\stanfordNLTK\jar\slf4j-api.jar", path_to_sihan_corpora_dict=r"E:\tools\stanfordNLTK\jar\data", path_to_model=r"E:\tools\stanfordNLTK\jar\data\pku.gz", path_to_dict=r"E:\tools\stanfordNLTK\jar\data\dict-chris6.ser.gz") result = segmenter.segment(mystr) print(result)
def __init__(self): file_path = path.realpath(__file__) dir_path = path.dirname(file_path) self.path_to_jar = path.join(dir_path, 'stanford-segmenter-3.9.2.jar') self.path_to_model = path.join(dir_path, 'data/ctb.gz') # pku.gz self.path_to_dict = path.join(dir_path, 'data/dict-chris6.ser.gz') self.path_to_sihan_corpora_dict = path.join(dir_path, 'data/') self.seg = StanfordSegmenter( path_to_jar=self.path_to_jar, java_class='edu.stanford.nlp.ie.crf.CRFClassifier', path_to_model=self.path_to_model, path_to_dict=self.path_to_dict, path_to_sihan_corpora_dict=self.path_to_sihan_corpora_dict)
def __init__(self): print('stanford segmenter init...') # stanford_corenlp_path = r'/media/mcislab/sdb1/home/mcislab/zwt/stanford_core_nlp' stanford_corenlp_path = r"D:\Desktop\stanford corenlp" self.segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'stanford-segmenter-3.9.1.jar'), path_to_slf4j=os.path.join(stanford_corenlp_path, 'slf4j-api-1.7.25.jar'), path_to_sihan_corpora_dict=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data'), path_to_model=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'pku.gz'), path_to_dict=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'dict-chris6.ser.gz'), sihan_post_processing='true' )
def get_stanford_segmenter(): if not os.path.isdir(STANFORD_SEGMENTER_DIR): download_stanford_segmenter() global STANFORD_SEGMENTER if not STANFORD_SEGMENTER: STANFORD_SEGMENTER = StanfordSegmenter( path_to_jar=STANFORD_SEGMENTER_JAR, path_to_sihan_corpora_dict=STANFORD_SIHAN_CORPORA_DICT, path_to_model=STANFORD_MODEL, path_to_dict=STANFORD_DICT, verbose=True ) return STANFORD_SEGMENTER
def Segmenter(segmenter_folder_name='', segmenter_jarname = '', segmenter_folder='', segmenter_jarpath='', segmenter_corpora='', segmenter_model='', segmenter_dictpath='', segmenter_slfpath=''): ### default_segmenter_folder_name='stanford-segmenter-2017-06-09' if len(segmenter_folder_name)==0: segmenter_folder_name = default_segmenter_folder_name ### default_segmenter_jarname = 'stanford-segmenter-3.8.0.jar' if len(segmenter_jarname) == 0: segmenter_jarname = default_segmenter_jarname ### default_segmenter_folder = os.path.join(os.path.expanduser('~'), 'Stanford NLP', segmenter_folder_name) if len(segmenter_folder)==0: segmenter_folder = default_segmenter_folder ### default_segmenter_jarpath = os.path.join(segmenter_folder, segmenter_jarname) if len(segmenter_jarpath) == 0: segmenter_jarpath = default_segmenter_jarpath ### default_segmenter_corpora = os.path.join(segmenter_folder, 'data') if len(segmenter_corpora) == 0: segmenter_corpora = default_segmenter_corpora ### default_segmenter_model = os.path.join(segmenter_folder, 'data', 'pku.gz') if len(segmenter_model) == 0: segmenter_model = default_segmenter_model ### default_segmenter_dictpath = os.path.join(segmenter_folder, 'data', 'dict-chris6.ser.gz') if len(segmenter_dictpath) == 0: segmenter_dictpath = default_segmenter_dictpath ### default_segmenter_slfpath = os.path.join(segmenter_folder, 'slf4j-api.jar') if len(segmenter_slfpath) == 0: segmenter_slfpath = default_segmenter_slfpath ### nltk.internals.config_java("") ###### segmenter = StanfordSegmenter(java_class="edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=segmenter_jarpath, path_to_sihan_corpora_dict=segmenter_corpora, path_to_model=segmenter_model, path_to_dict=segmenter_dictpath, path_to_slf4j=segmenter_slfpath) # segmenter.default_config('zh') ###### return segmenter
def segment(): """ split a Chinese sentence into words :return: """ segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\stanford-segmenter-3.9.1.jar", path_to_slf4j=r"D:\Desktop\stanford corenlp\slf4j-api-1.7.25.jar", path_to_sihan_corpora_dict=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data", path_to_model=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data\pku.gz", path_to_dict=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data\dict-chris6.ser.gz", sihan_post_processing='true' ) # path to jar files should be changed # result = segmenter.segment(s) result = segmenter.segment_sents(["一个人在切西红柿", "这个把手该换了", "别把手放在我的肩膀上", "他正在量和服尺寸"]) print(result)
def stanford_seg(path, stage='train'): df_dir = os.path.join(path, '{}.csv'.format(stage)) data = pd.read_csv(df_dir, error_bad_lines=False, dtype=object) data = data.dropna(axis=0, how='any').reset_index(drop=True) data_dir = '/home/trueto/stanford_segmenter/' seg = StanfordSegmenter(path_to_jar=data_dir + 'stanford-segmenter.jar', java_class='edu.stanford.nlp.ie.crf.CRFClassifier', path_to_sihan_corpora_dict=data_dir + "data", path_to_model=data_dir + 'data/pku.gz', path_to_dict=data_dir + "data/dict-chris6.ser.gz") columns = data.columns for column in columns: if column in ['question1', 'question2']: column_file = os.path.join(path, 'cut', '{}_{}.txt'.format(stage, column)) data[column].to_csv(column_file, index=False) cut_file = os.path.join(path, 'cut', '{}_{}_cut.txt'.format(stage, column)) with open(cut_file, 'w') as f: f.write(seg.segment_file(column_file))
def __init__(self, language): self.language = language model = self.models.get(language) dic = self.dics.get(language) if not model: raise (Exception( "Unsupported language for tokenizer: {}".format(language))) # Initialize Stanford Tokenizer self.tm_tokenize = StanfordSegmenter( path_to_jar=os.path.join(stanford_tokenizer_home, 'stanford-segmenter-3.6.0.jar'), path_to_model=os.path.join(stanford_tokenizer_home, 'data', model), path_to_dict=os.path.join(stanford_tokenizer_home, 'data', dic), path_to_sihan_corpora_dict=os.path.join(stanford_tokenizer_home, 'data'), path_to_slf4j=os.path.join(stanford_tokenizer_home, 'slf4j-api.jar'))
# encoding: utf-8 import nltk from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tag import StanfordNERTagger segmenter = StanfordSegmenter( #分词依赖的jar包 path_to_jar= r"/home/jiangix/document/stanford-segmenter/stanford-segmenter.jar", path_to_slf4j=r"/home/jiangix/document/slf4j/slf4j-api.jar", #分词数据文件夹 path_to_sihan_corpora_dict= r"/home/jiangix/document/stanford-segmenter/data", #基于北大在2005backoof上提供的人名日报语料库 path_to_model=r"/home/jiangix/document/stanford-segmenter/data/pku.gz", path_to_dict= r"/home/jiangix/document/stanford-segmenter/data/dict-chris6.ser.gz") segmenter.default_config('zh') result = segmenter.segment(u'我喜欢学习编程') chi_tagger = StanfordNERTagger( model_filename= r"/home/jiangix/document/stanford-chinese-corenlp-models/chinese.misc.distsim.crf.ser.gz", path_to_jar=r"/home/jiangix/document/stanford-ner/stanford-ner.jar") for word, tag in chi_tagger.tag(result.split()): print(word, tag)
# find_entity_t = test.find_entity() # find_VP_t = test.firstVP() # test.drawTree() test.show(firstNP_t) # test.show(find_entity_t) # test.show(find_VP_t) # # test.show(find_entity_t) # test.show(firstMinNP_t) result = test.find_realtionship(firstNP_t) print(result) test.drawTree() # # # print(test.rel) # test.show(test.find_realtionship()) # 对比实验 chi_parser = StanfordParser(path_to_jar='../stanford-parser-full-2018-02-27/stanford-parser.jar', path_to_models_jar='../stanford-parser-full-2018-02-27/stanford-parser-3.9.1-models.jar', model_path='../stanford-parser-full-2018-02-27/stanford-parser-3.9.1-models/edu/stanford/nlp/models/lexparser/chinesePCFG.ser.gz') data_dir='../stanford-segmenter-2018-02-27/' segmenter = StanfordSegmenter(path_to_jar=data_dir+"stanford-segmenter-3.9.1.jar", path_to_sihan_corpora_dict=data_dir+"/data", path_to_model=data_dir+"/data/pku.gz", path_to_dict=data_dir+"/data/dict-chris6.ser.gz", java_class='edu.stanford.nlp.ie.crf.CRFClassifier', ) result=segmenter.segment(test_str) result_ls = result.split() ch_tree = list(chi_parser.parse(result_ls))[0] ch_tree.draw() # print(result)
# from nltk.tokenize.stanford_segmenter import StanfordSegmenter # segmenter = StanfordSegmenter(path_to_jar="stanford-segmenter-3.4.1.jar", path_to_sihan_corpora_dict="./data", path_to_model="./data/pku.gz", path_to_dict="./data/dict-chris6.ser.gz") # sentence = u"这是斯坦福中文分词器测试" # segmenter.segment(sentence) # # u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n' # segmenter.segment_file("test.simp.utf8") # # u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ... ##下载 # import nltk # nltk.download() from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter( path_to_jar="stanford-segmenter-3.6.0.jar", path_to_slf4j = "slf4j-api.jar", path_to_sihan_corpora_dict="./data", path_to_model="./data/pku.gz", path_to_dict="./data/dict-chris6.ser.gz") sentence = u"这是斯坦福中文分词器测试" segmenter.segment(sentence) # >>> u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n' segmenter.segment_file("test.simp.utf8") # >>> u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ... # 英文测试 # import nltk # text = 'i am a good boy.you are a bad girl' # sens = nltk.sent_tokenize(text) # print(sens) # words = [] # for sent in sens:
import os, time from nltk.tokenize.stanford_segmenter import StanfordSegmenter start = time.time() jar = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/stanford-segmenter.jar' api_jar = '/Users/sinansmac/Public/StanfordNLP/stanford-parser-full-2017-06-09/slf4j-api.jar' # dict = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/data/dict-chris6.ser.gz' seg = StanfordSegmenter(path_to_jar=jar, path_to_slf4j=api_jar) seg.default_config('zh') # sent = u'这是斯坦福中文分词器测试' # print(seg.segment(sent)) fp = "chinese.txt" tokenstr = seg.segment_file(fp) token_ls = list(tokenstr) print(len(token_ls), '\n', tokenstr, '\n', token_ls) # with open('chinese_tokens.txt', 'a') as writef: # for line in token_ls: # writef.write(line.rstrip().split()) # print(tokens, '\n', type(tokens)) # class 'str' end = time.time() print("process time:", round(end - start))
print "{}: {}".format(key, value) if __name__ == '__main__': examples = [] with open(pjoin(args.data_dir, args.data_file), 'rb') as f: for line in f: examples.append(line) if args.corpus == "gigaword_ch" and not args.char: print "segmenting each example for Chinese, could take a while" from nltk.tokenize.stanford_segmenter import StanfordSegmenter seg = StanfordSegmenter(path_to_slf4j=path_to_slf4j, path_to_jar=path_to_jar) seg.default_config('zh') # ==== Filtering ===== data_dist = {} filtered_examples = {} number_of_filtered_examples = 0 for i, ex in enumerate(examples): s1, s2, label = ex[:-1].split('\t') if args.corpus == 'gigaword_ch': s1 = s1.replace(' .', '。') # parser appended normal period s2 = s2.replace(' .', '。') if args.char and args.corpus == "gigaword_ch": # we presplit into chars
# 3.2.4版本无法使用standford的2015版本,3.2.1可以 # from nltk.tokenize import StanfordTokenizer # tokenizer = StanfordTokenizer(path_to_jar=r"E://StanfordNLTK/stanford-parser.jar") # sent = "我爱我的祖国,我也愿为全人类的幸福而奋斗终生。无论什么时刻,我都是人类的伙伴。" # print(tokenizer.tokenize(sent)) from nltk.tokenize.stanford_segmenter import StanfordSegmenter # from nltk.parse.corenlp import * # 3.2.4版本无法使用standford的2015版本,3.2.1可以 # CoreNLPServer(path_to_jar="/home/fonttian/NLP/stanford/") # seg = StanfordSegmenter(path_to_slf4j='/home/fonttian/NLP/stanford-corenlp/slf4j-api.jar') seg = StanfordSegmenter() seg.default_config('zh') sent = u'这是斯坦福中文分词器测试' sent2 = u'我爱我的祖国,我也愿为全人类的幸福而奋斗终生 。' print(seg.segment(sent)) # segmenter = StanfordSegmenter( # path_to_jar="D:/Anaconda3/envs/MLEnv/StanfordNLTK/stanford-segmenter.jar", # path_to_slf4j="D:/Anaconda3/envs/MLEnv/StanfordNLTK/slf4j-api.jar", # path_to_sihan_corpora_dict="D:/Anaconda3/envs/MLEnv/StanfordNLTK/data", # path_to_model="D:/Anaconda3/envs/MLEnv/StanfordNLTK/data/pku.gz", # path_to_dict="D:/Anaconda3/envs/MLEnv/StanfordNLTK/data/dict-chris6.ser.gz" # ) # str = "我在博客园开了一个博客,我的博客名叫伏草惟存,写了一些自然语言处理的文章。" # result = segmenter.segment(str) # print(type(result))
# -*- coding: utf-8 -*- from __future__ import unicode_literals from nltk.tokenize.stanford_segmenter import StanfordSegmenter seg = StanfordSegmenter( path_to_jar=r'D:\temp\stanford\stanford-segmenter.jar', path_to_slf4j=r'D:\temp\stanford\slf4j-api.jar', path_to_sihan_corpora_dict=r'D:\temp\stanford\data', path_to_model=r'D:\temp\stanford\data\ctb.gz', path_to_dict=r'D:\temp\stanford\data\dict-chris6.ser.gz', ) # sentence = "这是斯坦福中文分词器测试" sentence = "操你大爷,狗日的" res = seg.segment(sentence) print res # import jieba # ss = jieba.cut(sentence) # print ' '.join(list(ss)), type(ss)
documents, keywords, filenames = load_texts_from_directory(input_dir) print(documents[0]) print(keywords[0]) print(filenames[0]) with open('chinese_all.txt', 'w', encoding='utf-8') as f: for d in documents: f.write(d) f.write('\n###############################################################################\n') f.close() # Segments file JAVA_PATH = 'C:/java/bin/java.exe' os.environ['JAVA_HOME'] = JAVA_PATH segmenter = StanfordSegmenter( path_to_jar='stanford-segmenter-2018-02-27/stanford-segmenter-3.9.1.jar', path_to_sihan_corpora_dict='./stanford-segmenter-2018-02-27/data', java_class='edu.stanford.nlp.ie.crf.CRFClassifier', path_to_model='stanford-segmenter-2018-02-27/data/pku.gz', path_to_dict='stanford-segmenter-2018-02-27/data/dict-chris6.ser.gz') with open('segmented_chinese.txt', mode='w', encoding='utf-8') as f: print('Starting segmentation') result = segmenter.segment_file('chinese_all.txt') print('now writing') f.write(result) f.close() # Separates into list documents_seg = [] with open('segmented_chinese.txt', 'r', encoding='utf-8') as f: doc = '' for d in f.readlines():
jieba.set_dictionary(jiebadict) jieba.load_userdict(jiebauserdict) n = 1 for line in open(sourcefile): words = pseg.cut(line) txts(pos) n = n + 1 print('Done') elif (tokeniser == 'stanfordpku'): from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tokenize import StanfordTokenizer workingdirectory = os.getcwd() segmenter = StanfordSegmenter( path_to_jar=os.path.join(workingdirectory, 'stanford-segmenter.jar'), path_to_slf4j=os.path.join(workingdirectory, 'slf4j-api.jar'), path_to_sihan_corpora_dict=os.path.join(workingdirectory, 'data'), path_to_model=os.path.join(workingdirectory, 'data', 'pku.gz'), path_to_dict=os.path.join(workingdirectory, 'data', 'dict-chris6.ser.gz')) tokenizer = StanfordTokenizer( path_to_jar=os.path.join(workingdirectory, 'stanford-parser.jar')) n = 1 for line in open(sourcefile): token = segmenter.segment(line) words = tokenizer.tokenize(token) with open('%s%s.txt' % (prefix, n), "w", encoding='utf-8') as resultfile: resultwrite = csv.writer(resultfile) for word in words: resultwrite.writerow([word]) n = n + 1
# -*- coding: utf-8 -*- # !/usr/bin/python import nltk #加载NLTK book模块 from nltk.book import * #text1; from nltk.tokenize.stanford_segmenter import StanfordSegmenter ''' path_to_jar: 用来定位jar包,本程序分词依赖stanford-segmenter.jar(其它所有Stanford NLP接口都有path_to_jar参数) path_to_slf4j: 用来定位slf4j-api.jar,作用于分词 path_to_sihan_corpora_dict: 设定为stanford-segmenter-2015-12-09.zip解压后目录中的data目录,data目录下有两个可用模型pkg.gz 和ctb.gz,需要注意的是,使用Stanford-Sementer进行中文分词后,其返回结果并不是list,而是一个字符中,各个汉语词江在其中被空格分隔开. ''' segmenter = StanfordSegmenter( path_to_jar=r"目录/stanford-segmenter.jar", path_to_slf4j=r"slf4j-api.jar", path_to_sihan_corpora_dict=r"jar/data", path_to_model=r"data/pku.gz", path_to_dict=r"data/dict-chris6.ser.gz", ) str = "我在博客园开了一个博客,我的博客名叫简单生活,写了一些关于服务端处理文章。" ''' 这个等相关的jar包下载完成后进行处理...... '''
#coding:UTF-8 from nltk.tokenize.stanford_segmenter import StanfordSegmenter segmenter = StanfordSegmenter( path_to_jar= r"D:\StanfordNLP\stanford-segmenter\stanford-segmenter-3.6.0.jar", path_to_slf4j=r"D:\StanfordNLP\stanford-segmenter\slf4j-api.jar", path_to_sihan_corpora_dict=r"D:\StanfordNLP\stanford-segmenter\data", path_to_model=r"D:\StanfordNLP\stanford-segmenter\data\pku.gz", path_to_dict=r"D:\StanfordNLP\stanford-segmenter\data\dict-chris6.ser.gz") str = u"我在博客园开了一个博客,我的博客名叫伏草惟存,写了一些自然语言处理的文章。" result = segmenter.segment(str) print result
'helpers', "englishPCFG.ser.gz") model_de_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers', "germanPCFG.ser.gz") jar_model_de_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers/stanford-corenlp-full-2016-10-31') model_cn_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers', "chinesePCFG.ser.gz") model_ch_lex_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers', "chineseFactored.ser.gz") segmenter = StanfordSegmenter( path_to_jar=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers', "stanford-segmenter.jar"), path_to_sihan_corpora_dict=os.path.join( os.path.dirname(os.path.realpath(__file__)), 'helpers', "stanford-segmenter-2015-12-09/data"), path_to_model=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'helpers', "stanford-segmenter-2015-12-09/data/pku.gz"), path_to_dict=os.path.join( os.path.dirname(os.path.realpath(__file__)), 'helpers', "stanford-segmenter-2015-12-09/data/dict-chris6.ser.gz")) #parser_en = StanfordParser(model_path = model_en_path) #parser_de = StanfordParser(model_path = model_de_path) #parser_cn = StanfordParser(model_path = model_cn_path) #parser_ch_lex = StanfordParser(model_path = model_ch_lex_path) dep_parser_en = StanfordDependencyParser(model_path=model_en_path) dep_parser_de = StanfordDependencyParser(model_path=model_de_path) dep_parser_cn = StanfordDependencyParser(model_path=model_cn_path)
需要设定 lyric_document = 专辑名 volume_of_word 词汇量, 如词汇量=600, 即保留出现频率最多的,600个词汇。 分词得到的参数文件(.npz)在data文件夹内: '专辑名_format_sen.npz' """ __author__ = 'Yunpeng Liu' import pandas as pd import os import numpy as np from nltk.tokenize.stanford_segmenter import StanfordSegmenter # 初始化segmenter segmenter = StanfordSegmenter( path_to_jar="./stanford-segmenter-2014-08-27/stanford-segmenter-3.4.1.jar", path_to_sihan_corpora_dict="./stanford-segmenter-2014-08-27/data", path_to_model="./stanford-segmenter-2014-08-27/data/pku.gz", path_to_dict="./stanford-segmenter-2014-08-27/data/dict-chris6.ser.gz") # 3类词语的特殊标签 unknown_token = "UNKNOWN_TOKEN" sentence_start_token = "SENTENCE_START" sentence_end_token = "SENTENCE_END" volume_of_word = 700 # 需要进行学习的词汇量 TODO: 人工设定 lyric_document = 'Fantasy' # 歌词文件名 TODO: 人工设定 # 词语和句子向量中 数字的 相互转化向量 word2num = {} num2word = [] word_dict = {}
#coding:utf8 import os import sys import logging from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tag import StanfordPOSTagger java_path = "C:\\Program Files\\Java\\jdk1.8.0_73\\bin\\java.exe" os.environ['JAVAHOME'] = java_path segmenter = StanfordSegmenter( path_to_jar= "E:\\lib\\stanford-segmenter-2017-06-09\\stanford-segmenter-3.8.0.jar", path_to_slf4j="E:\\lib\\stanford-segmenter-2017-06-09\\slf4j-api.jar", path_to_sihan_corpora_dict="E:\\lib\\stanford-segmenter-2017-06-09\\data", path_to_model="E:\\lib\\stanford-segmenter-2017-06-09\\data\\pku.gz", path_to_dict= "E:\\lib\\stanford-segmenter-2017-06-09\\data\\dict-chris6.ser.gz") postagger = StanfordPOSTagger( path_to_jar= "E:\\lib\\stanford-postagger-full-2017-06-09\\stanford-postagger.jar", model_filename= 'E:\\lib\\stanford-postagger-full-2017-06-09\\models\\chinese-distsim.tagger', ) def pos_to_sequence(sent, segmenter=segmenter, postagger=postagger): seg_sent = segmenter.segment(sent)
from nltk.tokenize.stanford_segmenter import StanfordSegmenter seg = StanfordSegmenter(path_to_slf4j='/YOUR_PATH/slf4j-api.jar') seg = StanfordSegmenter() seg.default_config('zh') sent = u'这是斯坦福中文分词器测试' print(seg.segment(sent))
#coding:utf-8 from nltk.tokenize.stanford_segmenter import StanfordSegmenter #中文分词 segmenter=StanfordSegmenter( path_to_jar="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/stanford-segmenter-3.5.2.jar", path_to_slf4j="/home/hsiao/Develops/nlp/stanford-corenlp-full-2016-10-31/slf4j-api.jar", path_to_sihan_corpora_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data", path_to_model="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/pku.gz", path_to_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/dict-chris6.ser.gz" ) str="我在我在博客园开了一个博客。" print (segmenter.segment(str)) #英文分词 from nltk.tokenize import StanfordTokenizer tokenizer=StanfordTokenizer(path_to_jar=r"/home/hsiao/Develops/nlp/stanford-parser-full-2016-10-31/stanford-parser.jar") sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." print (tokenizer.tokenize(sent)) #中文命名实体识别 from nltk.tag import StanfordNERTagger chi_tagger=StanfordNERTagger(model_filename=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/classifiers/chinese.misc.distsim.crf.ser.gz' ,path_to_jar=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/stanford-ner.jar') print (chi_tagger.tag('四川省 成都 信息 工程 大学 我 在 博客 园 开 了 一个 博客 , 我 的 博客 名叫 伏 草 惟 存 , 写 了 一些 自然语言 处理 的 文章 。\r\n'.split()))
# #sinica_treebank.tagged_words() # #sinica_treebank.parsed_sents()[15].draw() #from nltk.parse import stanford #s = '你好'.decode('utf-8') # #print s #parser = stanford.StanfordParser(path_to_jar='/home/alex/Software/nltk/stanford-parser.jar',path_to_models_jar='/home/alex/Software/nltk/stanford-parser-3.6.0-models.jar') #print parser.raw_parse_sents(s) from nltk.tokenize.stanford_segmenter import StanfordSegmenter import os segementerPath = '/media/Backup/nltk/stanford-segmenter-2015-12-09/' os.environ['CLASSPATH'] = segementerPath segmenter = StanfordSegmenter( path_to_jar=segementerPath + 'stanford-segmenter-3.6.0.jar', path_to_sihan_corpora_dict=segementerPath + 'data', path_to_model=segementerPath + 'data/pku.gz', path_to_dict=segementerPath + 'data/dict-chris6.ser.gz') sentence = u"这是斯坦福中文分词器测试" segmenter.segment(sentence) #segmenter.segment_file('test.simp.utf8') #outfile = open('outfile', 'w') #result = segmenter.segment(sentence) #outfile.write(result.encode('UTF-8')) #outfile.close()