def segment_en(texts, flag_keep_number=False): tk = StanfordTokenizer() results = {} for text in texts: if flag_keep_number: words = tk.tokenize(text) else: words = map(replace_number, tk.tokenize(text)) segmented = ' '.join(words).lower() results[text] = segmented return results
def tokenize_q(qa, phase): qas = len(qa) MyTokenizer = StanfordTokenizer() for i, row in enumerate(tqdm(qa)): row['question_toked'] = MyTokenizer.tokenize( row['question'].lower())[:14] if i % 50000 == 0: json.dump(qa, open('vqa_' + phase + '_toked_' + str(i) + '.json', 'w')) if i == qas - 1: json.dump(qa, open('vqa_' + phase + '_toked.json', 'w'))
def data(): with open("wonderland.txt", "r", encoding="utf-8-sig") as file: return [ word.lower() for word in StanfordTokenizer( path_to_jar= r"C:\stanford-postagger-2016-10-31\stanford-postagger.jar", options={ "normalizeParentheses": "false", "normalizeOtherBrackets": "false" }).tokenize(file.read()) ]
def __init__(self, task_queue, result_queue): multiprocessing.Process.__init__(self) self.task_queue = task_queue self.result_queue = result_queue self.tokenizer = StanfordTokenizer(options={"ptb3Escaping": True}) print '%s: Loading pickles...' % self.name self.map_word_index = pickle.load( open('preproc/map_word_index.pkl', 'r')) print '%s: Done.' % self.name
def Tokenize_stopwords_stemmer(texts): #print time() #用斯坦福的分词采用这一段,用普通分词时不用这个 #tokenize Str_texts = texts[0] print os.getcwd() #tokenizer = StanfordTokenizer(path_to_jar=r"/Users/apple/Documents/tools/stanford-parser-full-2015-04-20/stanford-parser.jar") tokenizer = StanfordTokenizer(path_to_jar=r"stanford-parser.jar") texts_tokenized = tokenizer.tokenize(Str_texts) #输入必须是字符串 #print time() p2 = r'.+[-_\./"].+' pa2 = re.compile(p2) texts_filtered = [] for document in texts_tokenized: if document in pa2.findall(document): if document.find('_') > -1: texts_filtered = texts_filtered + document.split('_') elif document.find('-') > -1: texts_filtered = texts_filtered + document.split('-') elif document.find('.') > -1: texts_filtered = texts_filtered + document.split('.') else: texts_filtered.append(document) #print time() p1 = r'[-@<#$%^&*].+' pa1 = re.compile(p1) p3 = r'.+">' pa3 = re.compile(p3) english_stopwords = stopwords.words('english') #得到停词 english_punctuations = [ ',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%', '\n', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '<', '>', '/', '\"', '\'', '{', '}', '!', '~', '`', '$', '^', '/*', '*/', '/**', '**/', '**', '-', '_', '+', '=', r'-?-', r'@?' ] #得到标点 texts_filtered0 = [] for document in texts_filtered: if document in pa1.findall(document) or document in pa3.findall( document ) or document == '' or document == "''" or document == "``" or document in english_stopwords or document in english_punctuations: pass else: texts_filtered0.append(document) #print time() porter = nltk.PorterStemmer() texts_Stemmered = [porter.stem(t) for t in texts_filtered0] #列表类型 #print time() return texts_Stemmered #返回一个列表
def readwordarr(isTokenize=True): posWords = [] negWords = [] stopwords = getstopword() if isTokenize: tokenizer = StanfordTokenizer() with open(negfilepath, 'r', encoding='utf-8') as sentences: arr = tokenizer.tokenize(sentences.read()) for line in arr: linearr = line.split() wordset = set() for word in linearr: if word in stopwords: continue wordset.add(word) negWords.append(list(wordset)) with open(posfilepath, 'r', encoding='utf-8') as sentences: arr = tokenizer.tokenize(sentences.read()) for line in arr: linearr = line.split() wordset = set() for word in linearr: if word in stopwords: continue wordset.add(word) posWords.append(list(wordset)) else: with open(negfilepath, 'r', encoding='utf-8') as sentences: lines = sentences.readlines() for line in lines: linearr = line.split() wordset = set() for word in linearr: if word in stopwords: continue wordset.add(word) negWords.append(list(wordset)) with open(posfilepath, 'r', encoding='utf-8') as sentences: lines = sentences.readlines() for line in lines: linearr = line.split() wordset = set() for word in linearr: if word in stopwords: continue wordset.add(word) posWords.append(list(wordset)) return posWords, negWords
def _get_sentence_embeddings(sentences, ngram='bigrams', model='concat_wiki_twitter'): """ Returns a numpy matrix of embeddings for one of the published models. It handles tokenization and can be given raw sentences. Arguments: - ngram: 'unigrams' or 'bigrams' - model: 'wiki', 'twitter', or 'concat_wiki_twitter' - sentences: a list of raw sentences ['Once upon a time', 'This is another sentence.', ...] """ wiki_embeddings = None twitter_embbedings = None tokenized_sentences_NLTK_tweets = None tokenized_sentences_SNLP = None if model == "wiki" or model == 'concat_wiki_twitter': tknzr = StanfordTokenizer(SNLP_TAGGER_JAR, encoding='utf-8') s = ' <delimiter> '.join( sentences) #just a trick to make things faster tokenized_sentences_SNLP = tokenize_sentences(tknzr, [s]) tokenized_sentences_SNLP = tokenized_sentences_SNLP[0].split( ' <delimiter> ') assert (len(tokenized_sentences_SNLP) == len(sentences)) if ngram == 'unigrams': wiki_embeddings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_SNLP, \ MODEL_WIKI_UNIGRAMS, FASTTEXT_EXEC_PATH) else: wiki_embeddings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_SNLP, \ MODEL_WIKI_BIGRAMS, FASTTEXT_EXEC_PATH) if model == "twitter" or model == 'concat_wiki_twitter': tknzr = TweetTokenizer() tokenized_sentences_NLTK_tweets = tokenize_sentences(tknzr, sentences) if ngram == 'unigrams': twitter_embbedings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_NLTK_tweets, \ MODEL_TWITTER_UNIGRAMS, FASTTEXT_EXEC_PATH) else: twitter_embbedings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_NLTK_tweets, \ MODEL_TWITTER_BIGRAMS, FASTTEXT_EXEC_PATH) if model == "twitter": return twitter_embbedings elif model == "wiki": return wiki_embeddings elif model == "concat_wiki_twitter": return np.concatenate((wiki_embeddings, twitter_embbedings), axis=1) sys.exit(-1)
def tokenize_stopwords_stemmer(texts): Str_texts = texts[0] # tokenizer = StanfordTokenizer(path_to_jar=r"/Users/apple/Documents/tools/stanford-parser-full-2015-04-20/stanford-parser.jar") tokenizer = StanfordTokenizer( path_to_jar=r"C:\Users\zw\Desktop\stanford-parser.jar") # path_to_jar: 用来定位jar包,r是防止字符转义的,如果路径中出现'\t'的话 不加r的话\t就会被转义 而加了'r'之后'\t'就能保留原有的样子 java_path = 'E:soft/Java/jdk1.8.0_121/bin/java.exe' os.environ['JAVAHOME'] = java_path texts_tokenized = tokenizer.tokenize(Str_texts) # 输入必须是字符串,进行分词 # print(texts_tokenized) p1 = r'[-@<#$%^&*].+' pa1 = re.compile(p1) # re.compile()函数,将正则表达式的字符串形式编译为Pattern实例,然后使用Pattern实例处理文本并获得匹配结果(一个Match实例) texts_filtered0 = [document for document in texts_tokenized if not document in pa1.findall(document)] p2 = r'.+[-_\/].+' # 将r'.+[-_\./].+'改为r'.+[-_\/].+',可以保留数字间的句号,比如保留3.1.2这样的格式 pa2 = re.compile(p2) texts_filtered = [] for document in texts_filtered0: if document in pa2.findall(document): if document.find('_') > -1: # split():拆分字符串。通过指定分隔符对字符串进行切片,并返回分割后的字符串列表(list) texts_filtered = texts_filtered + document.split('_') elif document.find('-') > -1: texts_filtered = texts_filtered + document.split('-') elif document.find('.') > -1: texts_filtered = texts_filtered + document.split('.') elif document.find('/') > -1: texts_filtered = texts_filtered + document.split('/') else: texts_filtered.append(document) texts_filtered = [document for document in texts_filtered if document != '' and document != "''" and document != "``"] # 过滤掉空格,单引号和-- # # stopwords # english_stopwords =stopwords.words('english') # 得到停词 # texts_filtered_stopwords = [document for document in texts_filtered if not document in english_stopwords] # 过滤掉停词 english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%', '\n', '||', '<', '>', '/', '\"', '\'', '{', '}', '!', '~', '`', '0', '$', '^', '/*', '*/', '/**', '**/', '**', '-', '_', '__', '|', '+', '=', r'-?-', r'@?'] # 得到标点 texts_filtered = [document for document in texts_filtered if not document in english_punctuations] # 过滤掉标点 return texts_filtered
def __init__(self, user_dict=None): self.conf_io = conf.load("io") self.conf_corenlp = conf.load("stanford_corenlp") self.conf_embedding = conf.load("embedding") conf_tokenizer = self.conf_corenlp["tokenizer"] conf_postagger = self.conf_corenlp["postagger"] prefix = self.conf_corenlp["prefix"] self.enTokenizer = StanfordTokenizer( path_to_jar=prefix + conf_tokenizer["path_to_jar"] ) self.zh_tagger = StanfordPOSTagger( prefix + conf_postagger["tagger_zh"], path_to_jar=prefix + conf_postagger["path_to_jar"] ) self.en_tagger = StanfordPOSTagger( prefix + conf_postagger["tagger_en"], path_to_jar=prefix + conf_postagger["path_to_jar"] )
def get_sentence_embeddings(sentences, ngram='uni'): tknzr = StanfordTokenizer(SNLP_TAGGER_JAR, encoding='utf-8') s = ' <delimiter> '.join(sentences) #just a trick to make things faster tokenized_sentences_SNLP = tokenize_sentences(tknzr, [s]) tokenized_sentences_SNLP = tokenized_sentences_SNLP[0].split( ' <delimiter> ') if len(tokenized_sentences_SNLP) != len(sentences): print('SENT2VEC TOKENIZATION FAILED') tokenized_sentences_SNLP = sentences #assert(len(tokenized_sentences_SNLP) == len(sentences)) if ngram == 'uni': embeddings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_SNLP, \ MODEL_TORONTOBOOKS_UNIGRAMS, FASTTEXT_EXEC_PATH) elif ngram == 'bi': embeddings = get_embeddings_for_preprocessed_sentences(tokenized_sentences_SNLP, \ MODEL_TORONTOBOOKS_BIGRAMS, FASTTEXT_EXEC_PATH) else: raise (NotImplementedError) return embeddings
def par_tokenize(text_list, clean_html=False, tokenizer="twitter", remove_reps=True, spell_correct=True): if tokenizer == "stanford": tolkenizer_obj = StanfordTokenizer() elif tokenizer == "twitter": tolkenizer_obj = TweetTokenizer() else: tolkenizer_obj = StringTokenizer() import multiprocessing as mp from functools import partial pool = mp.Pool(NUM_PROC) tolkenize_func = partial(__tolkenize_text_blob, clean_html=clean_html, remove_reps=remove_reps, spell_correct=spell_correct, tolkenizer_obj=tolkenizer_obj) token_list = pool.map(tolkenize_func, text_list) return token_list
def stanford_tokenizer(str): tokenizer = StanfordTokenizer( path_to_jar= 'D:/software/stanford-parser-full-3.7/stanford-parser-3.7.0-models.jar' ) # sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." return tokenizer.tokenize(str) # if __name__=='__main__': # sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." # result = stanford_tokenizer(sent) # print(result) # st = StanfordPOSTagger('english-bidirectional-distsim.tagger') # from nltk.tokenize import StanfordTokenizer # s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." # StanfordTokenizer().tokenize(s) # s = "The colour of the wall is blue." # StanfordTokenizer(options={"americanize": True}).tokenize(s)
def tokenize_and_save_corpus(corpus_filename, new_filename): with open(corpus_filename, 'r') as f: corpus_str = f.read() tokenized = StanfordTokenizer().tokenize(corpus_str) lowered = [w.lower() for w in tokenized] num = r'(?<!\S)(\d*\.?\d+|\d{1,3}(,\d{3})*(\.\d+)?)(?!\S)' number_words = {} new_words = [] for word in lowered: if word in number_words: new_words.extend(number_words[word]) else: numbers = re.findall(num, word) if numbers: number = numbers[0][0] nwords = word_numbers(number) number_words[word] = nwords new_words.extend(nwords) else: new_words.append(word) with open(new_filename, 'w') as f: f.write(' '.join(new_words).encode('utf-8'))
def tokenize(text_list, clean_html=False, tokenizer="twitter", remove_reps=True, spell_correct=True): if tokenizer == "stanford": tolkenizer_obj = StanfordTokenizer() elif tokenizer == "twitter": tolkenizer_obj = TweetTokenizer() else: tolkenizer_obj = StringTokenizer() token_list = [] for text in text_list: if clean_html: text = BeautifulSoup(text).get_text() if remove_reps: text = re.sub(r'(.)\1{2,}', r'\1\1', text) tokens = tolkenizer_obj.tokenize(text) if spell_correct: tokens = [spell(t) for t in tokens] token_list.append(tokens) return token_list
def __init__(self, **kwargs): self.conf_io = conf.load("io") self.conf_corenlp = conf.load("stanford_corenlp") self.conf_embedding = conf.load("embedding") conf_segmenter = self.conf_corenlp["segmenter"] conf_tokenizer = self.conf_corenlp["tokenizer"] conf_postagger = self.conf_corenlp["postagger"] prefix = self.conf_corenlp["prefix"] self.segmenter = StanfordSegmenter( path_to_jar=prefix + conf_segmenter["path_to_jar"], path_to_sihan_corpora_dict=prefix + conf_segmenter["path_to_sihan_corpora_dict"], path_to_model=prefix + conf_segmenter["path_to_model"], path_to_dict=prefix + conf_segmenter["path_to_dict"], path_to_slf4j=prefix + conf_segmenter["path_to_slf4j"], encoding=conf_segmenter["encoding"]) self.enTokenizer = StanfordTokenizer(path_to_jar=prefix + conf_tokenizer["path_to_jar"]) self.zh_tagger = StanfordPOSTagger( prefix + conf_postagger["tagger_zh"], path_to_jar=prefix + conf_postagger["path_to_jar"]) self.en_tagger = StanfordPOSTagger( prefix + conf_postagger["tagger_en"], path_to_jar=prefix + conf_postagger["path_to_jar"]) self.frequency = defaultdict(int) pynlpir.open() pynlpir.nlpir.ImportUserDict(conf.load("pynlpir")["user_dict"], Overwrite=False) try: self.excluded_docs = kwargs["excluded_docs"] except: self.excluded_docs = [""] # experimental features self.f_token_indexes = prefix + conf.load("pynlpir")["user_dict"]
def tokenize(text_rdd, clean_html=False, tokenizer="twitter", remove_reps=True, spell_correct=True): if tokenizer == "stanford": tokenizer_obj = StanfordTokenizer() elif tokenizer == "twitter": tokenizer_obj = TweetTokenizer() else: tokenizer_obj = StringTokenizer() print("Processing {} tokns".format(text_rdd.count())) if (remove_reps): text_rdd = text_rdd.map( lambda text: re.sub(r'(.)\1{2,}', r'\1\1', text)) if clean_html: text_rdd = text_rdd.map(lambda text: BeautifulSoup(text).get_text()) tokens_rdd = text_rdd.map(lambda text: TweetTokenizer().tokenize(text)) if spell_correct: tokens_rdd = tokens_rdd.map(lambda tokens: [spell(t) for t in tokens]) #tokens_rdd = tokens_rdd.map(lambda tokens: [t for t in tokens]) return tokens_rdd
#lines = f.read().encode('utf-8').split('</text>') #for index, line in enumerate(lines): # remove leading and trailing whitespace lines = file.encode('utf-8').split('</text>') for line in lines: newline = '' try: if "<text xml:space=\"preserve\">" in line and "#REDIRECT" not in line: newline = line[line.find("<text xml:space=\"preserve\">") + len("<text xml:space=\"preserve\">"):] if guess_language(newline) == 'en': s = re.sub( '[^A-Za-z0-9\s.,\'\";?$%+-:!]+', '@', re.sub('\d', '0', newline).replace('[', ' ').replace( ']', ' ').replace('}', ' ').replace('{', ' ')) s2 = StanfordTokenizer().tokenize(s) s3 = [word.encode('ascii') for word in s2] charCounter = 0 tokenCounter = 0 sentStart = 0 deleteThese = [] for index, token in enumerate(s3): if token == '.': if charCounter < 20 or tokenCounter < 5: deleteThese.append([sentStart, index]) charCounter = 0 tokenCounter = 0 sentStart = index + 1 else: charCounter += len(token) tokenCounter += 1
# -*- coding: utf-8 -*- from nltk.tokenize import StanfordTokenizer import time def is_ascii(s): return all(ord(c) < 128 for c in s) last_time = time.time() line_buffer = '' with open('WestburyLab.Wikipedia.Corpus.txt') as infp, open( 'TokenizedCorpus.txt', 'w') as outfp: for e, line in enumerate(infp): if (e + 1) % 10000 == 0: line_buffer = StanfordTokenizer().tokenize(line_buffer) try: outfp.write(' '.join(line_buffer) + '\n') except: for i in xrange(len(line_buffer)): if not is_ascii(line_buffer[i]): line_buffer[i] = '<UNK>' outfp.write(' '.join(line_buffer) + '\n') line_buffer = '' print e + 1, '/ 30749930', float( e + 1) / 30749930, time.time() - last_time if line.strip() == '': continue line_buffer += (line + ' <br> ')
java_path = "C:/Program Files/Java/jre1.8.0_131/bin/java.exe" parser_path = "D:/stanford-parser-full-2016-10-31/stanford-parser.jar" models_path = "D:/stanford-parser-full-2016-10-31/stanford-parser-3.7.0-models.jar" engPCFG_path = "edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz" import os os.environ['JAVA_HOME'] = java_path from nltk.tokenize import StanfordTokenizer tokenizer = StanfordTokenizer(parser_path) from nltk.parse.stanford import StanfordDependencyParser parser = StanfordDependencyParser(parser_path, models_path, engPCFG_path) from nltk.corpus import wordnet import nltk from nltk.tree import Tree from nltk.corpus.reader.wordnet import Lemma from nltk.corpus import semcor from nltk.corpus import wordnet noun = set(['NN', 'NNS', 'NNP', 'NNPS']) verb = set(['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']) adjective = set(['JJ', 'JJR', 'JJS']) adverb = set(['RB', 'RBR', 'RBS']) substantive = noun | verb | adjective | adverb corp = semcor.sents() tags = semcor.tagged_sents(tag = 'sem')
from nltk.tag.stanford import StanfordNERTagger, StanfordPOSTagger from nltk.tokenize import StanfordTokenizer from wordsegment import load, segment CUR_DIRECTORY = '/home/wmq/Desktop/DeepText/StanfordNLP' SEGMENT_PATH = CUR_DIRECTORY + '/stanford-segmenter-3.8.0.jar' NER_MODEL_PATH = CUR_DIRECTORY + '/english.all.3class.distsim.crf.ser.gz' NER_JAR_PATH = CUR_DIRECTORY + '/stanford-ner.jar' POS_MODEL_PATH = CUR_DIRECTORY + '/english-left3words-distsim.tagger' POS_JAR_PATH = CUR_DIRECTORY + '/stanford-postagger.jar' ner_tagger = StanfordNERTagger(NER_MODEL_PATH, NER_JAR_PATH, java_options='') pos_tagger = StanfordPOSTagger(POS_MODEL_PATH, POS_JAR_PATH, java_options='') tokenizer = StanfordTokenizer(SEGMENT_PATH) load() s = "@user nah pretty sure it's jackson's great jokes" ws = tokenizer.tokenize(s) print(' '.join(ws)) # print (' '.join(segment('#happythankgiving'))) # s = 'i got to to go formal with my best friend @ phi mu at jsu'.split() # ner_sent = ner_tagger.tag(s) # pos_sent = pos_tagger.tag(s) # print (ner_sent) # print (pos_sent)
def en_standseg(sent): tokenizer = StanfordTokenizer( path_to_jar=r"E:\tools\stanfordNLTK\jar\stanford-parser.jar") print(tokenizer.tokenize(sent))
def stanford_tokenize(s): return StanfordTokenizer().tokenize(s)
from lexnlp import is_stanford_enabled from lexnlp.nlp.en.tokens import STOPWORDS, get_lemma_list from lexnlp.config.stanford import STANFORD_POS_PATH __author__ = "ContraxSuite, LLC; LexPredict, LLC" __copyright__ = "Copyright 2015-2019, ContraxSuite, LLC" __license__ = "https://github.com/LexPredict/lexpredict-lexnlp/blob/master/LICENSE" __version__ = "0.2.7" __maintainer__ = "LexPredict, LLC" __email__ = "*****@*****.**" # Setup Stanford POS configuration try: STANFORD_POS_FILE = os.path.join(STANFORD_POS_PATH, "stanford-postagger.jar") STANFORD_TOKENIZER = StanfordTokenizer(path_to_jar=STANFORD_POS_FILE) STANFORD_DEFAULT_TAG_MODEL = os.path.join( STANFORD_POS_PATH, "models", "english-bidirectional-distsim.tagger") STANFORD_TAGGER = StanfordPOSTagger(STANFORD_DEFAULT_TAG_MODEL, STANFORD_POS_FILE) except LookupError: STANFORD_TOKENIZER = STANFORD_TAGGER = None def check_stanford(): if not is_stanford_enabled(): raise RuntimeError( "USE_STANFORD is set to False. No Stanford functionality available." ) if not STANFORD_TOKENIZER: raise RuntimeError("USE_STANFORD is set to True."
segmenter=StanfordSegmenter( path_to_jar="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/stanford-segmenter-3.5.2.jar", path_to_slf4j="/home/hsiao/Develops/nlp/stanford-corenlp-full-2016-10-31/slf4j-api.jar", path_to_sihan_corpora_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data", path_to_model="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/pku.gz", path_to_dict="/home/hsiao/Develops/nlp/stanford-segmenter-2015-04-20/data/dict-chris6.ser.gz" ) str="我在我在博客园开了一个博客。" print (segmenter.segment(str)) #英文分词 from nltk.tokenize import StanfordTokenizer tokenizer=StanfordTokenizer(path_to_jar=r"/home/hsiao/Develops/nlp/stanford-parser-full-2016-10-31/stanford-parser.jar") sent = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks." print (tokenizer.tokenize(sent)) #中文命名实体识别 from nltk.tag import StanfordNERTagger chi_tagger=StanfordNERTagger(model_filename=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/classifiers/chinese.misc.distsim.crf.ser.gz' ,path_to_jar=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/stanford-ner.jar') print (chi_tagger.tag('四川省 成都 信息 工程 大学 我 在 博客 园 开 了 一个 博客 , 我 的 博客 名叫 伏 草 惟 存 , 写 了 一些 自然语言 处理 的 文章 。\r\n'.split())) #英文命名实体识别 from nltk.tag import StanfordNERTagger eng_tagger=StanfordNERTagger(model_filename=r'/home/hsiao/Develops/nlp/stanford-ner-2016-10-31/classifiers/english.all.3class.distsim.crf.ser.gz'
# -*- coding:utf-8 -*- from nltk.tokenize import StanfordTokenizer from nltk.tokenize.stanford_segmenter import StanfordSegmenter from nltk.tag import StanfordPOSTagger from nltk.parse.stanford import StanfordDependencyParser from nltk.parse.stanford import StanfordParser import pickle as pkl # 英文分词 tokenizer = StanfordTokenizer( path_to_jar=r"D:\stanford-parser-full-2016-10-31\stanford-parser.jar") # # sentence2 = "Micron hasn't declared its first quarterly profit for three years, but he has been dead." # sentence = "Jim Williams director of the US VISIT project said that by the middle of November many arriving passengers in Atlanta will be fingerprinted and photographed" # # print(tokenizer.tokenize(sentence2)) # 英文词性标注 # 词性标注的标签说明:http://www.comp.leeds.ac.uk/amalgam/tagsets/upenn.html # eng_tagger = StanfordPOSTagger( # path_to_jar=r"D:\stanford-postagger-full-2016-10-31\stanford-postagger.jar", # model_filename=r"D:\stanford-postagger-full-2016-10-31\models\english-bidirectional-distsim.tagger" # ) # print("词性标注结果", eng_tagger.tag(sentence.split())) # 句法分析 # eng_parser = StanfordParser( # path_to_jar=r"D:\stanford-parser-full-2016-10-31\stanford-parser.jar", # path_to_models_jar=r"D:\stanford-parser-full-2016-10-31\stanford-parser-3.7.0-models.jar",
def __init__(self, classifier, jar_file, field_to_process, output_field): self.classifier = classifier self.jar_file = jar_file self.field_to_process = field_to_process self.output_field = output_field self.tokenizer = StanfordTokenizer(path_to_jar=self.jar_file).tokenize
elif token == '-RCB-': token = '}' return token def tokenize_sentences(tknzr, sentences, to_lower=True): """Arguments: - tknzr: a tokenizer implementing the NLTK tokenizer interface - sentences: a list of sentences - to_lower: lowercasing or not """ return [tokenize(tknzr, s, to_lower) for s in sentences] fileName = sys.argv[1] SNLP_TAGGER_JAR = "/home/pgupta/stanford-postagger.jar" sentences = [] with open(fileName, 'r') as fileinput: for line in fileinput: sentences.append(line) tknzr = StanfordTokenizer(SNLP_TAGGER_JAR, encoding='utf-8') s = ' <delimiter> '.join(sentences) tokenized_sentences_SNLP = tokenize_sentences(tknzr, [s]) tokenized_sentences_SNLP = tokenized_sentences_SNLP[0].split(' <delimiter> ') for sentence in tokenized_sentences_SNLP: print(sentence)
import re, nltk, sys from nltk.tokenize import StanfordTokenizer tokenizer = StanfordTokenizer( r'../common/stanford-postagger-2015-04-20/stanford-postagger.jar') for line in open("TEST_FILE.txt"): m = re.match(r'^([0-9]+)\s"(.+)"$', line.strip()) if m is not None: txtfile = open("test/%s.txt" % m.group(1), 'w') annfile = open("test/%s.ann" % m.group(1), 'w') line = m.group(2) text = [] t = line.split("<e1>") text.append(t[0]) e1start = len(t[0]) t = t[1].split("</e1>") e1 = t[0] text.append(t[0]) e1end = len(t[0]) + e1start t = t[1].split("<e2>") text.append(t[0]) e2start = len(t[0]) + e1end t = t[1].split("</e2>") text.append(t[0]) e2 = t[0] e2end = len(t[0]) + e2start text.append(t[1]) text = " ".join(tokenizer.tokenize("".join(text))) txtfile.write(text) txtfile.write("\n") offset = 0
abs_path = dirname(abspath(__file__)) # STANFORD TOOLS #------------------------------------------------------------------------------- # paths stanford_pos_dir = abs_path + '/stanford-postagger-2015-12-09/' postag_modelfile = stanford_pos_dir + 'models/english-bidirectional-distsim.tagger' postag_jar = stanford_pos_dir + 'stanford-postagger.jar' stanford_parser_dir = abs_path + '/stanford-parser-full-2015-12-09/' parser_eng_model = stanford_parser_dir + 'edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz' parser_models_jar = stanford_parser_dir + "stanford-parser-3.6.0-models.jar" parser_jar = stanford_parser_dir + "stanford-parser.jar" # Instances: st_tagger = StanfordPOSTagger(model_filename=postag_modelfile, path_to_jar=postag_jar) st_tknzr = StanfordTokenizer(path_to_jar=postag_jar) st_parser = StanfordParser(model_path=parser_eng_model, path_to_models_jar=parser_models_jar, path_to_jar=parser_jar) # BIKEL #------------------------------------------------------------------------------- # paths bikel_dir = abs_path + '/dbparser/' bk_parser_path = bikel_dir + 'bin/parse' bk_settings = bikel_dir + 'settings/collins.properties' bk_parser_model = bikel_dir + 'bikel/wsj-02-21.obj.gz' # cmd = bikel_parser+" 400"+bk_settings+" "+bk_parser_model + inputfile
info['share_indicator'] = np.nan info['num'] = np.nan info['person'] = np.nan info['num_person'] = np.nan # In[ ]: share_num = {} #token_dict = {} pos_dict = {} ent_dict = {} share_num_all = {} # In[ ]: tokenizer = StanfordTokenizer() pos = StanfordPOSTagger('english-bidirectional-distsim.tagger') parser = StanfordParser( model_path=u'edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz') dp = StanfordDependencyParser( model_path=u'edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz') ner = StanfordNERTagger('english.conll.4class.distsim.crf.ser.gz') # # 2. Define functions # ## 2.1 Joblib paralell cumputing function for loops