def splitAllWord(typeOfDataset="dev"):
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
segmenter = StanfordSegmenter()
segmenter.default_config('zh')
maxCount = 2000000
pathOfDev = "dataset/task1/%s.tsv" % typeOfDataset
dfOfDev = pd.read_csv(pathOfDev, delimiter="\t")
pathOfNewDev = "%s_split.tsv" % typeOfDataset
count = 0
with open(pathOfNewDev, "w", encoding='utf-8') as fw:
for row in dfOfDev.iterrows():
if count >= maxCount:
break
if count % 100 == 0:
print("[%s]count = %s" % (typeOfDataset, count))
label = row[1]['label']
fw.write(str(label))
fw.write("\t")
sentence = row[1]['text_a']
segmentOfSentence = segmenter.segment(sentence)
for word in segmentOfSentence.split():
fw.write(word)
fw.write(" ")
fw.write("\n")
count += 1
def segment(labels, reviews):
segmented = []
print('Creating BOW')
# seg = StanfordSegmenter('../../datasets/data-hauyi/stanford-segmenter-2018-10-16')
os.environ[
"STANFORD_SEGMENTER"] = '../datasets/data-hauyi/stanford-segmenter-2018-10-16'
seg = StanfordSegmenter(
'../datasets/data-hauyi/stanford-segmenter-2018-10-16/stanford-segmenter-3.9.2.jar'
)
seg.default_config('zh', )
count = 0
file_out = open('reviews.txt', 'a+')
for i in range(len(reviews)):
# print(i)
s = seg.segment(reviews[i])
l = labels[i]
# print(s)
line = str(l) + ' ' + s
file_out.write(line)
segmented.append(s)
# print('Tokenize: ')
# print(seg.tokenize(s))
count = count + 1
# if count > 5:
# break
print('Count: ', count)
return (segmented)
# encoding: utf-8
import nltk
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
from nltk.tag import StanfordNERTagger
segmenter = StanfordSegmenter(
#分词依赖的jar包
path_to_jar=
r"/home/jiangix/document/stanford-segmenter/stanford-segmenter.jar",
path_to_slf4j=r"/home/jiangix/document/slf4j/slf4j-api.jar",
#分词数据文件夹
path_to_sihan_corpora_dict=
r"/home/jiangix/document/stanford-segmenter/data",
#基于北大在2005backoof上提供的人名日报语料库
path_to_model=r"/home/jiangix/document/stanford-segmenter/data/pku.gz",
path_to_dict=
r"/home/jiangix/document/stanford-segmenter/data/dict-chris6.ser.gz")
segmenter.default_config('zh')
result = segmenter.segment(u'我喜欢学习编程')
chi_tagger = StanfordNERTagger(
model_filename=
r"/home/jiangix/document/stanford-chinese-corenlp-models/chinese.misc.distsim.crf.ser.gz",
path_to_jar=r"/home/jiangix/document/stanford-ner/stanford-ner.jar")
for word, tag in chi_tagger.tag(result.split()):
print(word, tag)
import os, time
from nltk.tokenize.stanford_segmenter import StanfordSegmenter
start = time.time()
jar = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/stanford-segmenter.jar'
api_jar = '/Users/sinansmac/Public/StanfordNLP/stanford-parser-full-2017-06-09/slf4j-api.jar'
# dict = '/Users/sinansmac/Public/StanfordNLP/stanford-segmenter-2017-06-09/data/dict-chris6.ser.gz'
seg = StanfordSegmenter(path_to_jar=jar, path_to_slf4j=api_jar)
seg.default_config('zh')
# sent = u'这是斯坦福中文分词器测试'
# print(seg.segment(sent))
fp = "chinese.txt"
tokenstr = seg.segment_file(fp)
token_ls = list(tokenstr)
print(len(token_ls), '\n', tokenstr, '\n', token_ls)
# with open('chinese_tokens.txt', 'a') as writef:
# for line in token_ls:
# writef.write(line.rstrip().split())
# print(tokens, '\n', type(tokens)) # class 'str'
end = time.time()
print("process time:", round(end - start))