def __init__(self, name): print("Starting new Tensorflow session...") self.session = tf.Session() print("Loading pipeline modules...") self.tokenizer = segmenter.load_model(name) self.tagger_pos = pos_tagger.load_model(name) # class tagger_pos self.tagger_ner = ner_tagger.load_model(name) # class tagger_ner
# -*- coding:utf-8 -*- import jieba from deepnlp import ner_tagger if __name__ == '__main__': tagger = ner_tagger.load_model(lang='en') test = 'UK London and United States are both big city.' wordList = ' '.join(jieba.cut(test)).split() print(wordList) taggedZip = tagger.predict(wordList) itemReStr = '' for (w, t) in taggedZip: itemReStr = itemReStr + w + '/' + t + ' ' print(itemReStr)
testtesttest#!/usr/bin/python # -*- coding:utf-8 -*- from __future__ import unicode_literals # compatible with python3 unicode import deepnlp deepnlp.download('ner') # download the NER pretrained models from github if installed from pip from deepnlp import ner_tagger tagger = ner_tagger.load_model(name = 'zh') # Base LSTM Based Model tagger.load_dict("zh_o2o") # Change to other dict text = "北京 望京 最好吃 的 黑椒 牛排 在哪里" words = text.split(" ") # Use the prefix dict and merging function to combine separated words tagging = tagger._predict_ner_tags_dict(words, merge = True) print ("DEBUG: NER tagger zh_o2o dictionary") for (w,t) in tagging: pair = w + "/" + t print (pair) #北京/city #望京/area #最好吃/nt #的/nt #黑椒牛排/dish #在哪里/nt # Word Sense Disambuguition
#!/usr/bin/python # -*- coding:utf-8 -*- from __future__ import unicode_literals # compatible with python3 unicode import deepnlp deepnlp.download( 'ner' ) # download the NER pretrained models from github if installed from pip from deepnlp import ner_tagger # Example 1. Change to other dict tagger = ner_tagger.load_model( name='zh_o2o') # Base LSTM Based Model + zh_o2o dictionary text = "北京 望京 最好吃 的 小龙虾 在 哪里" words = text.split(" ") tagging = tagger.predict(words, tagset=['city', 'area', 'dish']) for (w, t) in tagging: pair = w + "/" + t print(pair) #Result #北京/city #望京/area #最好吃/nt #的/nt #小/nt #龙虾/dish #在/nt #哪里/nt
#coding:utf-8 from __future__ import unicode_literals import sys,os import codecs from deepnlp import segmenter from deepnlp import pos_tagger # module: pos_tagger from deepnlp import ner_tagger # module: ner_tagger # Create new tagger instance tagger_pos = pos_tagger.load_model(lang = 'zh') tagger_ner = ner_tagger.load_model(lang = 'zh') BASE_DIR = os.path.dirname(os.path.abspath(__file__)) # concatenate tuples into one string "w1/t1 w2/t2 ..." def _concat_tuples(tagging): TOKEN_BLANK = " " wl = [] # wordlist for (x, y) in tagging: wl.append(x + "/" + y) concat_str = TOKEN_BLANK.join(wl) return concat_str # read input file docs = [] file = codecs.open(os.path.join(BASE_DIR, 'docs_test.txt'), 'r', encoding='utf-8') for line in file: line = line.replace("\n", "").replace("\r", "") docs.append(line)
print(e) ## pos from deepnlp import pos_tagger try: tagger = pos_tagger.load_model( name='en') # Loading English model, lang code 'en' tagger = pos_tagger.load_model( name='zh') # Loading English model, lang code 'en' except Exception as e: print("DEBUG: ERROR Found...") print(e) ## ner from deepnlp import ner_tagger try: my_tagger = ner_tagger.load_model(name='zh') my_tagger = ner_tagger.load_model(name='zh_o2o') my_tagger = ner_tagger.load_model(name='zh_entertainment') except Exception as e: print("DEBUG: ERROR Found...") print(e) ## parse from deepnlp import nn_parser try: parser = nn_parser.load_model(name='zh') except Exception as e: print("DEBUG: ERROR Found...") print(e)
import deepnlp deepnlp.download('ner') from deepnlp import segmenter from deepnlp import ner_tagger tokenizer = segmenter.load_model(name='zh') tagger = ner_tagger.load_model(name='zh') file_r = open('./data/person_resume.txt', 'r') file_w = open('./data/person_resume02.txt', 'w') for line in file_r: text = line.rstrip() words = tokenizer.seg(text) tagging = tagger.predict(words) for (w, t) in tagging: pair = w + ' ' + t file_w.write(pair) file_w.close() file_r.close()
def __init__(self, lang): print("Starting new Tensorflow session...") self.session = tf.Session() print("Loading pipeline modules...") self.tagger_pos = pos_tagger.load_model(lang) # class tagger_pos self.tagger_ner = ner_tagger.load_model(lang) # class tagger_ner
deepnlp.download("pos", "zh_finance") try: pos_tagger.load_model("zh_finance") except Exception as e: print (e) from deepnlp import ner_tagger try: deepnlp.download("ner", "zh_finance") except Exception as e: print (e) deepnlp.register_model("ner", "zh_finance") deepnlp.download("ner", "zh_finance") try: ner_tagger.load_model("zh_finance") except Exception as e: print (e) from deepnlp import nn_parser try: deepnlp.download("parse", "zh_finance") except Exception as e: print (e) deepnlp.register_model("parse", "zh_finance") deepnlp.download("parse", "zh_finance") try: nn_parser.load_model("zh_finance") except Exception as e: print (e)
# -*- coding:utf-8 -*- import deepnlp import jieba deepnlp.download('ner') # deepnlp.download() from deepnlp import ner_tagger tagger = ner_tagger.load_model(lang='zh') # Loading Chinese NER model exit(0) text = "习近平的妻子是彭丽媛" words = jieba.cut(text) print(" ".join(words).encode('utf-8')) print('---------------------------------') # tagging = tagger.predict(words) # for (w,t) in tagging: # str = w + "/" + t # print (str.encode('utf-8'))