def stanford_seg(path, stage='train'):
df_dir = os.path.join(path, '{}.csv'.format(stage))
data = pd.read_csv(df_dir, error_bad_lines=False, dtype=object)
data = data.dropna(axis=0, how='any').reset_index(drop=True)
data_dir = '/home/trueto/stanford_segmenter/'
seg = StanfordSegmenter(path_to_jar=data_dir + 'stanford-segmenter.jar',
java_class='edu.stanford.nlp.ie.crf.CRFClassifier',
path_to_sihan_corpora_dict=data_dir + "data",
path_to_model=data_dir + 'data/pku.gz',
path_to_dict=data_dir + "data/dict-chris6.ser.gz")
columns = data.columns
for column in columns:
if column in ['question1', 'question2']:
column_file = os.path.join(path, 'cut',
'{}_{}.txt'.format(stage, column))
data[column].to_csv(column_file, index=False)
cut_file = os.path.join(path, 'cut',
'{}_{}_cut.txt'.format(stage, column))
with open(cut_file, 'w') as f:
f.write(seg.segment_file(column_file))
class SegmentWorker:
def __init__(self):
file_path = path.realpath(__file__)
dir_path = path.dirname(file_path)
self.path_to_jar = path.join(dir_path, 'stanford-segmenter-3.9.2.jar')
self.path_to_model = path.join(dir_path, 'data/ctb.gz') # pku.gz
self.path_to_dict = path.join(dir_path, 'data/dict-chris6.ser.gz')
self.path_to_sihan_corpora_dict = path.join(dir_path, 'data/')
self.seg = StanfordSegmenter(
path_to_jar=self.path_to_jar,
java_class='edu.stanford.nlp.ie.crf.CRFClassifier',
path_to_model=self.path_to_model,
path_to_dict=self.path_to_dict,
path_to_sihan_corpora_dict=self.path_to_sihan_corpora_dict)
def seg_file(self, file_to_segment):
"""segment a file and return the result string"""
seg_result = self.seg.segment_file(file_to_segment)
translator = str.maketrans('', '', string.digits)
seg_result = seg_result.translate(translator)
seg_result = re.sub('[\\\\.!/_,$%^*(+\\"\']+|[+—！，：；。？、~@#￥%…&*（）]+',
'', seg_result)
# print(seg_result)
return seg_result
def seg_file2list(self, file_to_segment):
"""segment a text file and return array of tokens"""
seg_result = self.seg_file(file_to_segment)
# print(seg_result)
return seg_result.split()
def seg_file2file(self, origin_file, dest_file):
"""segment a text file and write result tokens to another file"""
seg_result = self.seg_file(origin_file)
seg_result = re.sub('\\s+', ' ', seg_result)
# print(seg_result)
with open(dest_file, 'w', encoding='UTF-8') as f:
f.write(seg_result)
# # u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ...
##下载
# import nltk
# nltk.download()
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
segmenter = StanfordSegmenter(
path_to_jar="stanford-segmenter-3.6.0.jar",
path_to_slf4j = "slf4j-api.jar",
path_to_sihan_corpora_dict="./data",
path_to_model="./data/pku.gz",
path_to_dict="./data/dict-chris6.ser.gz")
sentence = u"这是斯坦福中文分词器测试"
segmenter.segment(sentence)
# >>> u'\u8fd9 \u662f \u65af\u5766\u798f \u4e2d\u6587 \u5206\u8bcd\u5668 \u6d4b\u8bd5\n'
segmenter.segment_file("test.simp.utf8")
# >>> u'\u9762\u5bf9 \u65b0 \u4e16\u7eaa \uff0c \u4e16\u754c \u5404\u56fd ...
# 英文测试
# import nltk
# text = 'i am a good boy.you are a bad girl'
# sens = nltk.sent_tokenize(text)
# print(sens)
# words = []
# for sent in sens:
# words.append(nltk.word_tokenize(sent))
# for line in words:
# print(line)
#
# tags = []
# for tokens in words:
import os, time
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
start = time.time()
jar = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/stanford-segmenter.jar'
api_jar = '/Users/sinansmac/Public/StanfordNLP/stanford-parser-full-2017-06-09/slf4j-api.jar'
# dict = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/data/dict-chris6.ser.gz'
seg = StanfordSegmenter(path_to_jar=jar, path_to_slf4j=api_jar)
seg.default_config('zh')
# sent = u'这是斯坦福中文分词器测试'
# print(seg.segment(sent))
fp = "chinese.txt"
tokenstr = seg.segment_file(fp)
token_ls = list(tokenstr)
print(len(token_ls), '\n', tokenstr, '\n', token_ls)
# with open('chinese_tokens.txt', 'a') as writef:
# for line in token_ls:
# writef.write(line.rstrip().split())
# print(tokens, '\n', type(tokens)) # class 'str'
end = time.time()
print("process time:", round(end - start))
# from nltk.tokenize.stanford import
path = "D:/www/data/nlpsoftware/stanford-segmenter"
segmenter = StanfordSegmenter(
path_to_jar=path + "/stanford-segmenter.jar",
path_to_sihan_corpora_dict=path + "/data",
path_to_model=path + "/data/pku.gz",
path_to_dict=path + "/data/dict-chris6.ser.gz",
java_class='edu.stanford.nlp.ie.crf.CRFClassifier')
#
sentence = u"这是斯坦福中文分词器测试"
sentence = u"工信处女干事每月经过 下属 科室都要亲口交代24口交换机等技术性器件的安装工作"
segmenter.tokenize_sents(u"工信处")
result = segmenter.segment(sentence)
result2 = segmenter.segment_file(
"D:/www/data/nlpdata/icwb2-data/testing/pku_test.utf8")
clean_content = "D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content.txt"
# clean_content_out="D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content_out.txt"
# result3 = segmenter.segment_file(clean_content)
print(type(result2))
# with open(clean_content_out,'wb+') as f:
# f.writelines([(s+"\r\n").encode('utf8') for s in clean_content_out])
print(result2)
# outfile = open("D:/www/data/nlpsoftware/outfile.txt",'w')
# outfile.write(result)
# outfile.close()
#
# stanford_postagger="D:\\www\\data/nlpsoftware/stanford-postagger-full-2017-06-09\\stanford-postagger.jar"
# stanford_ner="D:\\www\\data/nlpsoftware/stanford-ner-2017-06-09\\stanford-ner.jar"
# classifiers="D:\\www\\data\\nlpsoftware\\stanford-ner\\classifiers\\"
f.write('\n###############################################################################\n')
f.close()
# Segments file
JAVA_PATH = 'C:/java/bin/java.exe'
os.environ['JAVA_HOME'] = JAVA_PATH
segmenter = StanfordSegmenter(
path_to_jar='stanford-segmenter-2018-02-27/stanford-segmenter-3.9.1.jar',
path_to_sihan_corpora_dict='./stanford-segmenter-2018-02-27/data',
java_class='edu.stanford.nlp.ie.crf.CRFClassifier',
path_to_model='stanford-segmenter-2018-02-27/data/pku.gz',
path_to_dict='stanford-segmenter-2018-02-27/data/dict-chris6.ser.gz')
with open('segmented_chinese.txt', mode='w', encoding='utf-8') as f:
print('Starting segmentation')
result = segmenter.segment_file('chinese_all.txt')
print('now writing')
f.write(result)
f.close()
# Separates into list
documents_seg = []
with open('segmented_chinese.txt', 'r', encoding='utf-8') as f:
doc = ''
for d in f.readlines():
if '###############################################################################' in d:
documents_seg.append(doc)
doc = ''
else:
doc = ' '.join([doc, d])
f.close()
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
from nltk.tokenize.stanford import CoreNLPTokenizer
# from nltk.tokenize.stanford import
path = "D:/www/data/nlpsoftware/stanford-segmenter"
segmenter = StanfordSegmenter(
path_to_jar=path + "/stanford-segmenter.jar",
path_to_sihan_corpora_dict=path + "/data",
path_to_model=path + "/data/pku.gz",
path_to_dict=path + "/data/dict-chris6.ser.gz",
java_class='edu.stanford.nlp.ie.crf.CRFClassifier'
)
#
# sentence = u"这是斯坦福中文分词器测试
# result = segmenter.segment(sentence)
# result2 = segmenter.segment_file("D:/www/data/nlpdata/icwb2-data/testing/pku_test.utf8")
clean_content="D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content.txt"
clean_content_out="D:\\www\\data\\Weibo Data\\Weibo Data\\nlp/clean_content_out.txt"
result3 = segmenter.segment_file(clean_content)
print(result3)
with open(clean_content_out,'wb+') as f:
f.write(result3.encode('utf8'))