def segment_sentences(sentence_list): segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'stanford-segmenter-3.9.1.jar'), path_to_slf4j=os.path.join(stanford_corenlp_path, 'slf4j-api-1.7.25.jar'), path_to_sihan_corpora_dict=os.path.join( stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data'), path_to_model=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'pku.gz'), path_to_dict=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'dict-chris6.ser.gz'), sihan_post_processing='true') result = segmenter.segment_sents(sentence_list) result = result.strip() segmented_list = re.split(os.linesep, result) if len(segmented_list[-1]) == 0: segmented_list = segmented_list[:-1] if len(segmented_list) != len(sentence_list): for i in range(len(segmented_list)): ss = ''.join(segmented_list[i].split()) if ss != sentence_list[i]: print(i, '|', segmented_list[i], '|', sentence_list[i]) # break print(len(segmented_list), len(sentence_list)) assert len(segmented_list) == len(sentence_list) return segmented_list
class StanfordTokenizer: """ class for segmenting Chinese sentences uses stanford segmenter 3.9.1 """ def __init__(self): stanford_corenlp_path = r'/media/mcislab/sdb1/home/mcislab/zwt/stanford_core_nlp' self.segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'stanford-segmenter-3.9.1.jar'), path_to_slf4j=os.path.join(stanford_corenlp_path, 'slf4j-api-1.7.25.jar'), path_to_sihan_corpora_dict=os.path.join( stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data'), path_to_model=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'pku.gz'), path_to_dict=os.path.join(stanford_corenlp_path, 'stanford-segmenter-2018-02-27', 'data', 'dict-chris6.ser.gz'), sihan_post_processing='true') def segment_sents(self, sentences): result = self.segmenter.segment_sents(sentences) result = result.strip() segmented_list = re.split(os.linesep, result) if len(segmented_list[-1]) == 0: segmented_list = segmented_list[:-1] print(len(segmented_list), len(sentences)) assert len(segmented_list) == len(sentences) return segmented_list def tokenize(self, captions_for_images): image_id_list = [] caption_list = [] for (image_id, captions) in captions_for_images.items(): for caption in captions: caption_list.append(caption['caption']) image_id_list.append(image_id) segmented_caption_list = self.segment_sents(caption_list) assert len(image_id_list) == len(caption_list) and len( caption_list) == len(segmented_caption_list) tokenized_captions_for_images = {} for i in range(len(image_id_list)): image_id = image_id_list[i] if image_id not in tokenized_captions_for_images: tokenized_captions_for_images[image_id] = [] tokenized_captions_for_images[image_id].append( segmented_caption_list[i]) return tokenized_captions_for_images
def segment(): """ split a Chinese sentence into words :return: """ segmenter = StanfordSegmenter( java_class=r"edu.stanford.nlp.ie.crf.CRFClassifier", path_to_jar=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\stanford-segmenter-3.9.1.jar", path_to_slf4j=r"D:\Desktop\stanford corenlp\slf4j-api-1.7.25.jar", path_to_sihan_corpora_dict=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data", path_to_model=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data\pku.gz", path_to_dict=r"D:\Desktop\stanford corenlp\stanford-segmenter-2018-02-27\data\dict-chris6.ser.gz", sihan_post_processing='true' ) # path to jar files should be changed # result = segmenter.segment(s) result = segmenter.segment_sents(["一个人在切西红柿", "这个把手该换了", "别把手放在我的肩膀上", "他正在量和服尺寸"]) print(result)
examples += filtered_examples[label] print "total number in produced dataset: {}".format(len(examples)) # now we word segment for Chinese if args.corpus == "gigaword_ch" and not args.char: s1_list, s2_list, labels = [], [], [] for ex in examples: s1, s2, label = ex.split('\t') s1_list.append(s1.decode('utf-8')) s2_list.append(s2.decode('utf-8')) labels.append(label) logging.info("s1, s2 collected, segmentation begins") s1_list = seg.segment_sents(s1_list) s1_list = s1_list.split('\n')[:-1] logging.info("s1 segmented") s2_list = seg.segment_sents(s2_list) s2_list = s2_list.split('\n')[:-1] logging.info("s2 segmented") examples = [] assert len(s1_list) == len(s2_list) == len(labels) for i in range(len(s1_list)): example_line = "\t".join([s1_list[i], s2_list[i], labels[i]]) # label has '\n' examples.append( example_line ) # no need to encode in utf-8 anymore, seg produces utf-8
class NLPCore: """ nlp processing including Stanford Word Segmenter, Stanford POS Tagger, Stanford Named Entity Recognizer and Stanford Parser """ def __init__(self): self.root_path = '../Models/stanfordNLP/' # word segmenter self.segmenter = StanfordSegmenter( path_to_jar=self.root_path + "stanford-segmenter.jar", path_to_slf4j=self.root_path + "log4j-over-slf4j.jar", path_to_sihan_corpora_dict=self.root_path + "segmenter/", path_to_model=self.root_path + "segmenter/pku.gz", path_to_dict=self.root_path + "segmenter/dict-chris6.ser.gz") # pos tagger self.posTagger = StanfordPOSTagger( self.root_path + 'pos-tagger/chinese-distsim.tagger', path_to_jar=self.root_path + "stanford-postagger.jar") # named entity recognizer self.nerTagger = StanfordNERTagger( self.root_path + 'ner/chinese.misc.distsim.crf.ser.gz', path_to_jar=self.root_path + 'stanford-ner.jar') self.parser = StanfordDependencyParser( model_path=self.root_path + 'lexparser/chinesePCFG.ser.gz', path_to_jar=self.root_path + 'stanford-parser.jar', path_to_models_jar=self.root_path + 'stanford-parser-3.7.0-models.jar', encoding='gbk') def split_sent_stanford(self, textPair): """ Stanford Word Segmenter, input should be raw text :return: also TextPair with raw string of results """ t1 = self.segmenter.segment(textPair.t1) t2 = self.segmenter.segment(textPair.t1) if DEBUG: print(t1, t2) return text_pair.TextPair(t1, t2, textPair.label) def split_sents_stanford(self, textPairs): """ Stanford Word Segmenter, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1 for textPair in textPairs] sents2 = [textPair.t2 for textPair in textPairs] split1 = self.segmenter.segment_sents(sents1).split('\n') split2 = self.segmenter.segment_sents(sents2).split('\n') rlist = [] for i in range(len(textPairs)): rlist.append( text_pair.TextPair(split1[i], split2[i], textPairs[i].label)) if DEBUG: print(split1[i], split2[i]) return rlist def split_sent_jieba(self, textPair): jieba.setLogLevel('INFO') ger1 = jieba.cut(textPair.t1) ger2 = jieba.cut(textPair.t2) t1 = ' '.join(ger1) t2 = ' '.join(ger2) return text_pair.TextPair(t1, t2, textPair.label) def pos_tag(self, textPair): """ Stanford POS Tagger, input should be splitted :return: also TextPair with raw string of results """ t1_s = textPair.t1.split() t2_s = textPair.t2.split() t1_tag = ' '.join([ele[1] for ele in self.posTagger.tag(t1_s)]) t2_tag = ' '.join([ele[1] for ele in self.posTagger.tag(t2_s)]) if DEBUG: print(t1_tag, t2_tag) return text_pair.TextPair(t1_tag, t2_tag, textPair.label) def pos_tag_pairs(self, textPairs): """ Stanford POS Tagger, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1.split() for textPair in textPairs] sents2 = [textPair.t2.split() for textPair in textPairs] tag1 = self.posTagger.tag_sents(sents1) tag2 = self.posTagger.tag_sents(sents2) rlist = [] for i in range(len(tag1)): t1_tag = ' '.join([ele[1] for ele in tag1[i]]) t2_tag = ' '.join([ele[1] for ele in tag2[i]]) rlist.append(text_pair.TextPair(t1_tag, t2_tag, textPairs[i].label)) if DEBUG: print(t1_tag, t2_tag) return rlist def ner_tag(self, textPair): """ Stanford Named Entity Recognizer, input should be splitted :return: also TextPair with raw string of results """ t1_s = textPair.t1.split() t2_s = textPair.t2.split() t1_ner = ' '.join( [ele[0] + '#' + ele[1] for ele in self.nerTagger.tag(t1_s)]) t2_ner = ' '.join( [ele[0] + '#' + ele[1] for ele in self.nerTagger.tag(t2_s)]) if DEBUG: print(t1_ner, t2_ner) return text_pair.TextPair(t1_ner, t2_ner, textPair.label) def ner_tag_pairs(self, textPairs): """ Stanford Named Entity Recognizer, input should be list of sents :return: also TextPair with raw string of results """ sents1 = [textPair.t1.split() for textPair in textPairs] sents2 = [textPair.t2.split() for textPair in textPairs] tag1 = self.nerTagger.tag_sents(sents1) tag2 = self.nerTagger.tag_sents(sents2) rlist = [] for i in range(len(tag1)): t1_ner = ' '.join([ele[0] + '#' + ele[1] for ele in tag1[i]]) t2_ner = ' '.join([ele[0] + '#' + ele[1] for ele in tag2[i]]) rlist.append(text_pair.TextPair(t1_ner, t2_ner, textPairs[i].label)) if DEBUG: print(t1_ner, t2_ner) return rlist def depen_parse(self, textPair): """ Stanford Dependency Parser, input should be splitted :return: also TextPair with raw string of results """ print([p.tree() for p in self.parser.raw_parse(textPair.t1)])