def nlp_preprocess(filepath_dict: dict, col: str, df=None, verbose: bool = True, overwrite_interim: bool = True) -> pd.DataFrame: def clean_doc(corpus): ''' generator function to read in docs from the file, and substitute and remove substrings ''' for doc in corpus: yield au_tu.remove_substrings(au_tu.clean_tokens( doc, tokens=to_replace_dict, whole_words_only=whole_words_only, ignore_case=ignore_case, ), to_remove_list=to_remove_list, whole_words_only=whole_words_only, ignore_case=ignore_case) def tokenize_entities(parsed_doc): txt = parsed_doc.text for ent in parsed_doc.ents: txt = txt[:ent.start_char] + ent.text.replace( ' ', '_') + txt[ent.end_char:] return txt def cleaned_doc_corpus(corpus): ''' generator function to use spaCy to parse docs, clean docs, tokenize named entities, and yield documents ''' for parsed_doc in nlp.pipe(clean_doc(corpus), batch_size=nlp_batch_size, n_threads=nlp_n_threads): yield tokenize_entities(parsed_doc) def punct_space_more(token): ''' helper function to eliminate tokens that are pure punctuation or whitespace or digits or only 1 character ''' return ( token.is_punct or token.is_space or token.is_digit or token.text == "'s" or token.lemma_ == '-PRON-' or # token.lemma_ == 'say' or # token.lemma_ == 'tell' or # token.lemma_ == 'be' or len(token.text) <= 1) def line_doc(filename): ''' generator function to read in docs from the file, un-escape the original line breaks in the text, and do additional cleaning ''' def hyp_to_us(doc): return re.sub(r'\b-\b', '_', doc) def remove_punct(doc): # keep: alphanumberic (w), spaces (s), single quote, underscore return re.sub(r'[^\w\s\'_]+', '', doc) # with codecs.open(filename, encoding='utf_8') as f: with smart_open(filename) as f: for doc in f: yield remove_punct(hyp_to_us(doc.decode())).replace( '\\n', '\n') def lemmatized_sentence_corpus(filename): ''' generator function to use spaCy to parse docs, lemmatize the text, and yield sentences ''' for parsed_doc in nlp.pipe(line_doc(filename), batch_size=nlp_batch_size, n_threads=nlp_n_threads): for sent in parsed_doc.sents: yield ' '.join([ token.lemma_ for token in sent if not punct_space_more(token) ]) if verbose: logger.info(f'Working on text from: {col}') # # debug - only getting from the sample dataframe here # df_phrased = df.loc[df[col].notnull(), ['tfa_master_uid', 'app_year', col]].sample(n=50).copy() df_phrased = df.loc[df[col].notnull(), ['tfa_master_uid', 'app_year', col]].copy() nlp = spacy.load('en', disable=[]) # clean text and tokenize entities if verbose: logger.info('Cleaning docs...') df_phrased[col] = list(cleaned_doc_corpus(df_phrased[col].values)) # remove 'the_' from NER tokens df_phrased[col] = df_phrased[col].apply( lambda x: ' '.join([re.sub('^the_', 'the ', y) for y in x.split()])) if verbose: logger.info('\tDone.') # create & open a new file in write mode if verbose: logger.info('Saving documents, one per line...') doc_count = 0 with codecs.open(filepath_dict['doc_txt_filepath'], 'w', encoding='utf_8') as doc_txt_file: for doc in df_phrased[[col]].apply(lambda x: ' '.join(x), axis=1).tolist(): # write the doc as a line in the new file # escape newline characters in the original doc text doc_txt_file.write(doc.replace('\n', '\\n') + '\n') doc_count += 1 if verbose: logger.info( f"Text from {doc_count:,} docs written to: {filepath_dict['doc_txt_filepath']}" ) nlp = spacy.load('en', disable=['ner']) # lemmatize and save sentences if overwrite_interim: if verbose: logger.info( f"Processing documents into unigram sentences: {filepath_dict['unigram_sentences_filepath']}" ) # with codecs.open(filepath_dict['unigram_sentences_filepath'], 'w', encoding='utf_8') as f: with smart_open(filepath_dict['unigram_sentences_filepath'], 'w') as f: for sentence in lemmatized_sentence_corpus( filepath_dict['doc_txt_filepath']): f.write(sentence + '\n') if verbose: logger.info('Done.') unigram_sentences = LineSentence( filepath_dict['unigram_sentences_filepath']) if verbose: logger.info('Unigram examples:') for unigram_sentence in it.islice(unigram_sentences, 10, 20): logger.info(u' '.join(unigram_sentence)) logger.info('=' * 30) if verbose: logger.info('Finding bigram phrases') # create the bigram model bigram = Phrases(unigram_sentences, min_count=phrase_min_count, threshold=phrase_threshold, max_vocab_size=phrase_max_vocab_size, progress_per=phrase_progress_per, scoring=phrase_scoring, common_terms=phrase_common_terms) bigram_model = Phraser(bigram) bigram_model.save(filepath_dict['bigram_model_filepath']) if verbose: logger.info( f"Saving bigram phrased sentences: {filepath_dict['bigram_sentences_filepath']}" ) # save bigram sentences with codecs.open(filepath_dict['bigram_sentences_filepath'], 'w', encoding='utf_8') as f: for unigram_sentence in unigram_sentences: bigram_sentence = u' '.join(bigram_model[unigram_sentence]) f.write(bigram_sentence + '\n') bigram_sentences = LineSentence( filepath_dict['bigram_sentences_filepath']) if verbose: logger.info('Bigram examples:') for bigram_sentence in it.islice(bigram_sentences, 10, 20): logger.info(u' '.join(bigram_sentence)) logger.info('=' * 30) if verbose: logger.info('Finding trigram phrases') # create the trigram model trigram = Phrases(bigram_sentences, min_count=phrase_min_count, threshold=phrase_threshold, max_vocab_size=phrase_max_vocab_size, progress_per=phrase_progress_per, scoring=phrase_scoring, common_terms=phrase_common_terms) trigram_model = Phraser(trigram) trigram_model.save(filepath_dict['trigram_model_filepath']) if verbose: logger.info( f"Saving trigram phrased sentences: {filepath_dict['trigram_sentences_filepath']}" ) # create trigram sentences with codecs.open(filepath_dict['trigram_sentences_filepath'], 'w', encoding='utf_8') as f: for bigram_sentence in bigram_sentences: trigram_sentence = u' '.join(trigram_model[bigram_sentence]) f.write(trigram_sentence + '\n') trigram_sentences = LineSentence( filepath_dict['trigram_sentences_filepath']) if verbose: logger.info('Trigram examples:') for trigram_sentence in it.islice(trigram_sentences, 10, 20): logger.info(u' '.join(trigram_sentence)) logger.info('=' * 30) if verbose: logger.info( f"Saving phrased docs using saved models: {filepath_dict['trigram_docs_filepath']}" ) # using saved models, write transformed text out to a new file, one doc per line with codecs.open(filepath_dict['trigram_docs_filepath'], 'w', encoding='utf_8') as f: for parsed_doc in nlp.pipe(line_doc(filepath_dict['doc_txt_filepath']), batch_size=nlp_batch_size, n_threads=nlp_n_threads): # removing punctuation and whitespace unigram_doc = [ token.lemma_ for token in parsed_doc if not punct_space_more(token) ] # apply the first-order and second-order phrase models bigram_doc = bigram_model[unigram_doc] trigram_doc = trigram_model[bigram_doc] # remove any remaining stopwords trigram_doc = [ term for term in trigram_doc if term not in nlp.Defaults.stop_words ] #extend the stop workds stop_words_extend = [ 'from', 'subject', 're', 'edu', 'use', 'not', 'would', 'say', 'could', '_', 'be', 'know', 'good', 'go', 'get', 'do', 'done', 'try', 'many', 'some', 'nice', 'thank', 'think', 'see', 'rather', 'easy', 'easily', 'lot', 'lack', 'make', 'want', 'seem', 'run', 'need', 'even', 'right', 'line', 'even', 'also', 'may', 'take', 'come' ] trigram_doc = [ term for term in trigram_doc if term not in stop_words_extended ] # write the transformed doc as a line in the new file trigram_doc = ' '.join(trigram_doc) f.write(trigram_doc + '\n') if verbose: logger.info('Done.') # put the text back in the dataframe trigram_docs = LineSentence(filepath_dict['trigram_docs_filepath']) if len([doc for doc in trigram_docs]) == df_phrased.shape[0]: for i, doc in enumerate(trigram_docs): df_phrased.iloc[i, df_phrased.columns.get_loc(col)] = ' '.join(doc) else: raise ValueError( 'Different number of processed and original documents') # save dataframe if verbose: logger.info('Saving NLP processed data: {}'.format( filepath_dict['filepath_out'])) df_phrased.to_csv(filepath_dict['filepath_out']) return df_phrased
# -*- coding: utf-8 -*- # train_word2vec_model.py用于训练模型 import logging import os.path import sys import multiprocessing from gensim.corpora import WikiCorpus from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence if __name__ == '__main__': program = os.path.basename(sys.argv[0]) logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) # check and process input arguments if len(sys.argv) < 4: print(globals()['__doc__'] % locals()) sys.exit(1) inp, outp1, outp2 = sys.argv[1:4] model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count()) # trim unneeded model memory = use(much) less RAM #model.init_sims(replace=True) model.save(outp1) model.wv.save_word2vec_format(outp2, binary=False)
# -*- coding: utf-8 -*- from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence import time # 训练模型 t0 = int(time.time()) sentences = LineSentence('wiki.zh.word.text') # size:词向量长度 # window:词向量上下文最大距离,默认为 5,对于一般的语料,这个值推荐在 [5, 10] 之间 # sg:如果设置为 0,则是 CBOW 模型,如果是 1,则是 Skip-Gram 模型,默认为 0 # hs:如果是 0,则是负采样(Negative Sampling),如果是 1,则是 Hierarchical Softmax,默认为 0 # negative:即使用 Negative Sampling 时负采样的个数,默认是 5,推荐在 [3, 10] 之间 # min_count:需要计算词向量的最小词频,这个值可以去掉一些很生僻的低频词,默认值为 5 # iter:随机梯度下降法中迭代的最大次数,默认是 5 # alpha:在随机梯度下降法中迭代的初始步长,默认是0.025 # min_alpha: 由于算法支持在迭代的过程中逐渐减小步长,min_alpha 给出了最小的迭代步长值。随机梯度下降中每轮的迭代步长可以由 iter、alpha、min_alpha 一起得出 model = Word2Vec(sentences, size=128, window=5, min_count=5, workers=4) print('训练耗时 %d s' % (int(time.time()) - t0)) # 保存模型 model.save('gensim_128')
# line = f.readline() # lastline="" # while line: # try: # if lastline!=line: # wf.write(line) # lastline = line # line = f.readline() # except UnicodeDecodeError: # print('error') # f.seek(2,1) # f.close() # wf.close() if mode_append == False: sentences = LineSentence(words_file) model = Word2Vec(sentences, size=150, hs=1, window=5) model.save(model_file) print(model.most_similar('天才')) else: # i=0 model = Word2Vec.load(model_file) print('load finish') model.build_vocab(corpus_file=words_file, update=True) # print(model2.corpus_total_words) # model2.build_vocab(corpus_file='./data/zh.seg.txt', update=True) print('build finish') model.train(corpus_file=words_file, total_examples=model.corpus_count, epochs=model.iter, total_words=model.corpus_total_words)
imp.reload(sys) origin_file = "../create_dataset/data/clotho_csv_files/clotho_captions_development.csv" output_file = "./target_development.txt" rd = '[\s+\.\!\/_,$%^*(+\"\')]+|[+——()?【】“”!,。?、::~@#¥%……&*()《》<>]+' print("prepare data") ori = codecs.open(origin_file, 'r', encoding='utf-8') out = codecs.open(output_file, 'w', encoding='utf-8') print("read files") text = pd.read_csv(origin_file, index_col=None)[[ 'caption_1', 'caption_2', 'caption_3', 'caption_4', 'caption_5' ]].values.tolist() out_text = [re.sub(rd, ' ', t.lower()) for caption in text for t in caption] out_text = '\n'.join(out_text) out.writelines(out_text) ori.close() out.close() print("Generated word vector") logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) sentences = LineSentence(output_file) model = Word2Vec(sentences, size=192, min_count=1, iter=1000, workers=8) model.train(sentences, total_examples=model.corpus_count, epochs=1000) model.save("./w2v_192.mod") print("Done")
#_*_coding:utf-8_*_ from gensim import models from gensim.models.word2vec import LineSentence import codecs import multiprocessing import logging import os logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) word = '../data/raw_data/word.csv' article = '../data/raw_data/article.csv' file_word = codecs.open(word, 'r', 'utf-8') model = models.Word2Vec(LineSentence(file_word), sg=0, size=192, window=5, min_count=5, workers=multiprocessing.cpu_count()) #model.save('../data/model/word.bin') model.wv.save_word2vec_format('../data/model/word/word.txt') file_article = codecs.open(article, 'r', 'utf-8') model = models.Word2Vec(LineSentence(file_article), sg=0, size=192, window=5, min_count=5, workers=multiprocessing.cpu_count())
#coding: utf8 import multiprocessing from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence sentences_file = '/home/parallels/dev/ai-nlp/lesson-five/cut_words_new' output_model_file = '/home/parallels/dev/ai-nlp/lesson-five/word_to_vec_model_new' #model = Word2Vec(LineSentence(sentences_file), window=5, workers=multiprocessing.cpu_count()) model = Word2Vec(LineSentence(sentences_file), size=200, window=5, min_count=5, workers=4) model.save(output_model_file)
def train_w2v(): with open('./data/reduced_zhwiki.txt', 'r', encoding='utf8') as f: # 使用gensim的Word2Vec类来生成词向量 model = Word2Vec(LineSentence(f), sg=0, size=192, window=5, min_count=5, workers=4) model.save('./data/zhwiki_news.word2vec')
config_pattern = "size{}window{}sg{}min_count{}negative{}iter{}" config_str = config_pattern.format(args.size, args.window, args.sg, args.min_count, args.negative, args.iter) outputfile1 = outputpath + config_str + ".model" outputfile2 = outputpath + config_str + ".vector" ############### end of config ################# logging.basicConfig(filename=config_str + '.log', filemode='w', format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) logger = logging.getLogger() logger.info("running train process in custom: %s" % args.train) model = Word2Vec( LineSentence(inpputfile), size=args.size, window=args.window, min_count=args. min_count, # with 0.35 billion corpus, #3000 can retain 9228 unique words workers=args.workers, # multiprocessing.cpu_count() #sample=args.sample, sg=args.sg, #hs=args.hs, negative=args. negative, # follow tensorflow's word2vec_optimized.py num_neg_samples 25 iter=args.iter) # trim unneeded model memory = use(much) less RAM # model.init_sims(replace=True) model.save(outputfile1)
def stool_simulator(total_epoch, special_epoch_count, restricted_vocab_name): # corpus_file = '/Users/zzcoolj/Code/GoW/data/training data/Wikipedia-Dumps_en_20170420_prep/AA/wiki_01.txt' corpus_file = 'input/enwiki-1G.txt' xlsx_path = 'output/test1G-vocab50000-stool-iter' + str(total_epoch) + '-first' + str(special_epoch_count) \ + 'EpochInitial-' + str(restricted_vocab_name) + '.xlsx' df = pd.DataFrame(columns=[ # word embeddings file name 'file name', # wordsim353 'wordsim353_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # simlex999 'simlex999_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # MTURK-771 'MTURK771_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # questions-words 'sem_acc', '#sem', 'syn_acc', '#syn', 'total_acc', '#total' ]) line_number_in_xlsx = 0 lr = 0.025 alphas = alpha_splitter(start=lr, epochs=total_epoch) print('alphas', alphas) # special starting epochs (final notIn) restricted_vocab = read_file_to_dict('../word_embeddings_evaluator/data/distinct-tokens/' + str(restricted_vocab_name) + '.txt') restricted_type = 1 params = { 'alpha': lr, 'min_alpha': alphas[special_epoch_count], 'size': 200, 'window': 5, 'iter': special_epoch_count, 'max_vocab_size': 50000, 'sample': 1e-4, 'sg': 1, # 1 for skip-gram 'hs': 0, # If 0, and negative is non-zero, negative sampling will be used. 'negative': 5, 'workers': 3, 'restricted_vocab': restricted_vocab, # [modified] ATTENTION: It must be a dictionary not a list! 'restricted_type': restricted_type # [modified] 0: train_batch_sg_original; 1: train_batch_sg_in; 2: train_batch_sg_notIn } print('special epochs half', special_epoch_count) gs_model = Word2Vec(LineSentence(corpus_file), **params) df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch' + str(special_epoch_count) + '-half') # special starting epochs (final in) print('special epochs entire', special_epoch_count) gs_model.restricted_type = 2 gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=gs_model.iter, start_alpha=lr, end_alpha=alphas[special_epoch_count]) line_number_in_xlsx += 1 df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch' + str(special_epoch_count) + '-entire') # original ending epochs print('roof epochs') gs_model.restricted_type = 0 gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=total_epoch-special_epoch_count, start_alpha=alphas[special_epoch_count], end_alpha=alphas[-1]) line_number_in_xlsx += 1 df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch' + str(total_epoch)) writer = pd.ExcelWriter(xlsx_path) df.to_excel(writer, 'Sheet1') writer.save()
def iteration_simulator(total_epoch, special_epoch_count, restricted_vocab_name, jumps): # corpus_file = '/Users/zzcoolj/Code/GoW/data/training data/Wikipedia-Dumps_en_20170420_prep/AA/wiki_01.txt' corpus_file = 'input/enwiki-1G.txt' xlsx_path = 'output/test1G-vocab50000-original-iter' + str(total_epoch) + '-last' + str(special_epoch_count) \ + 'EpochInitial-' + str(restricted_vocab_name) + '-jump'+''.join(str(x) for x in jumps)+'.xlsx' df = pd.DataFrame(columns=[ # word embeddings file name 'file name', # wordsim353 'wordsim353_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # simlex999 'simlex999_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # MTURK-771 'MTURK771_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # questions-words 'sem_acc', '#sem', 'syn_acc', '#syn', 'total_acc', '#total' ]) line_number_in_xlsx = 0 # epoch 0 lr = 0.025 alphas = alpha_splitter(start=lr, epochs=total_epoch) print('alphas', alphas) min_alpha = alphas[1] restricted_vocab = read_file_to_dict('../word_embeddings_evaluator/data/distinct-tokens/' + str(restricted_vocab_name) + '.txt') restricted_type = 0 params = { 'alpha': lr, 'min_alpha': min_alpha, 'size': 200, 'window': 5, 'iter': 0, # TODO NOW 'max_vocab_size': 50000, 'sample': 1e-4, 'sg': 1, # 1 for skip-gram 'hs': 0, # If 0, and negative is non-zero, negative sampling will be used. 'negative': 5, 'workers': 3, 'restricted_vocab': restricted_vocab, # [modified] ATTENTION: It must be a dictionary not a list! 'restricted_type': restricted_type # [modified] 0: train_batch_sg_original; 1: train_batch_sg_in; 2: train_batch_sg_notIn } print('cur_epoch', 0) gs_model = Word2Vec(LineSentence(corpus_file), **params) df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch0') gs_model.epochs = 1 # TODO NOW # # epoch 0.5 # gs_model.restricted_type = 2 # gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=gs_model.iter, # start_alpha=lr, end_alpha=min_alpha) # df.loc[1] = evaluate(gs_model.wv, 'X-iter0.5') # epoch 1+ # gs_model.restricted_type = 0 for cur_epoch in range(1, total_epoch-special_epoch_count): print('cur_epoch', cur_epoch) start_alpha = alphas[cur_epoch] end_alpha = alphas[cur_epoch+1] print('start_alpha', start_alpha) print('end_alpha', end_alpha) gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=gs_model.iter, start_alpha=start_alpha, end_alpha=end_alpha) line_number_in_xlsx += 1 df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch'+str(cur_epoch)) # # save common base model # write_to_pickle(gs_model, xlsx_path.split('.xlsx')[0]+'-base') for special_epoch in range(total_epoch-special_epoch_count, total_epoch): print('special epoch', special_epoch) start_alpha = alphas[special_epoch] end_alpha = alphas[special_epoch+1] print('start_alpha', start_alpha) print('end_alpha', end_alpha) # final special epochs 0.5 gs_model.restricted_type = 1 gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=gs_model.iter, start_alpha=start_alpha, end_alpha=end_alpha) line_number_in_xlsx += 1 df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch'+str(special_epoch)+'-half') # final special epochs final if special_epoch not in jumps: gs_model.restricted_type = 2 gs_model.train(LineSentence(corpus_file), total_examples=gs_model.corpus_count, epochs=gs_model.iter, start_alpha=start_alpha, end_alpha=end_alpha) line_number_in_xlsx += 1 df.loc[line_number_in_xlsx] = evaluate(gs_model.wv, 'epoch' + str(special_epoch)+'-entire') # # baseline (final original word2vec epochs) # gs_model_base = read_pickle(xlsx_path.split('.xlsx')[0] + '-base') # gs_model_base.restricted_type = 0 # for baseline_epoch in range(total_epoch - special_epoch_count, total_epoch): # print('baseline epoch', baseline_epoch) # start_alpha = alphas[baseline_epoch] # end_alpha = alphas[baseline_epoch + 1] # print('start_alpha', start_alpha) # print('end_alpha', end_alpha) # gs_model_base.train(LineSentence(corpus_file), total_examples=gs_model_base.corpus_count, epochs=gs_model_base.iter, # start_alpha=start_alpha, end_alpha=end_alpha) # line_number_in_xlsx += 1 # df.loc[line_number_in_xlsx] = evaluate(gs_model_base.wv, 'epoch' + str(baseline_epoch)+'-baseline') writer = pd.ExcelWriter(xlsx_path) df.to_excel(writer, 'Sheet1') writer.save()
# MTURK-771 'MTURK771_Pearson correlation', 'Pearson pvalue', 'Spearman correlation', 'Spearman pvalue', 'Ration of pairs with OOV', # questions-words 'sem_acc', '#sem', 'syn_acc', '#syn', 'total_acc', '#total' ]) for i in range(5): params = { 'alpha': 0.025, 'min_alpha': 0.0001, 'size': 200, 'window': 5, 'iter': 5, 'max_vocab_size': 50000, 'sample': 1e-4, 'sg': 1, # 1 for skip-gram 'hs': 0, # If 0, and negative is non-zero, negative sampling will be used. 'negative': 5, 'workers': 3, 'restricted_vocab': None, # [modified] ATTENTION: It must be a dictionary not a list! 'restricted_type': 0 # [modified] 0: train_batch_sg_original; 1: train_batch_sg_in; 2: train_batch_sg_notIn } gs_model = Word2Vec(LineSentence(corpus_file), **params) df.loc[i] = evaluate(gs_model.wv, str(i)) writer = pd.ExcelWriter(xlsx_path) df.to_excel(writer, 'Sheet1') writer.save()
def word2vec_training(text_file): sentences = LineSentence(text_file) model = Word2Vec(sentences, size=300, window=5, min_count=1, workers=16) model.wv.save("merge_with_unk.kv") # model.wv.save_word2vec_format("merge_with_unk_vector.txt", binary=False) return model
def _train_phrase_detection_model(self, input_filepath, output_filepath): sentences = LineSentence(input_filepath) model = Phraser(Phrases(sentences)) self._save_sentences(sentences, model, output_filepath) return model
def build_dataset(train_data_path, test_data_path): ''' 数据加载+预处理 :param train_data_path:训练集路径 :param test_data_path: 测试集路径 :return: 训练数据 测试数据 合并后的数据 ''' # 1.加载数据 train_df = pd.read_csv(train_data_path) test_df = pd.read_csv(test_data_path) print('train data size {},test data size {}'.format( len(train_df), len(test_df))) # 2. 空值剔除 train_df.dropna(subset=['Report'], inplace=True) train_df.fillna('', inplace=True) test_df.fillna('', inplace=True) # 3.多线程, 批量数据处理 train_df = parallelize(train_df, sentences_proc) test_df = parallelize(test_df, sentences_proc) # 4. 合并训练测试集合 train_df['merged'] = train_df[['Question', 'Dialogue', 'Report']].apply(lambda x: ' '.join(x), axis=1) test_df['merged'] = test_df[['Question', 'Dialogue']].apply(lambda x: ' '.join(x), axis=1) merged_df = pd.concat([train_df[['merged']], test_df[['merged']]], axis=0) print('train data size {},test data size {},merged_df data size {}'.format( len(train_df), len(test_df), len(merged_df))) # 5.保存处理好的 训练 测试集合 train_df = train_df.drop(['merged'], axis=1) test_df = test_df.drop(['merged'], axis=1) train_df.to_csv(train_seg_path, index=None, header=False) test_df.to_csv(test_seg_path, index=None, header=False) # 6. 保存合并数据 merged_df.to_csv(merger_seg_path, index=None, header=False) # 7. 训练词向量 print('start build w2v model') wv_model = Word2Vec(LineSentence(merger_seg_path), size=embedding_dim, sg=1, workers=8, iter=wv_train_epochs, window=5, min_count=5) # 8. 分离数据和标签 train_df['X'] = train_df[['Question', 'Dialogue']].apply(lambda x: ' '.join(x), axis=1) test_df['X'] = test_df[['Question', 'Dialogue']].apply(lambda x: ' '.join(x), axis=1) # 训练集 验证集划分 X_train, X_val, y_train, y_val = train_test_split( train_df['X'], train_df['Report'], test_size=0.002, # 8W*0.002 ) X_train.to_csv(train_x_seg_path, index=None, header=False) y_train.to_csv(train_y_seg_path, index=None, header=False) X_val.to_csv(val_x_seg_path, index=None, header=False) y_val.to_csv(val_y_seg_path, index=None, header=False) test_df['X'].to_csv(test_x_seg_path, index=None, header=False) # 9. 填充开始结束符号,未知词填充 oov, 长度填充 # 使用GenSim训练得出的vocab vocab = wv_model.wv.vocab # 训练集X处理 # 获取适当的最大长度 train_x_max_len = get_max_len(train_df['X']) test_X_max_len = get_max_len(test_df['X']) X_max_len = max(train_x_max_len, test_X_max_len) train_df['X'] = train_df['X'].apply( lambda x: pad_proc(x, X_max_len, vocab)) # 测试集X处理 # 获取适当的最大长度 test_df['X'] = test_df['X'].apply(lambda x: pad_proc(x, X_max_len, vocab)) # 训练集Y处理 # 获取适当的最大长度 train_y_max_len = get_max_len(train_df['Report']) train_df['Y'] = train_df['Report'].apply( lambda x: pad_proc(x, train_y_max_len, vocab)) # 10. 保存pad oov处理后的,数据和标签 train_df['X'].to_csv(train_x_pad_path, index=None, header=False) train_df['Y'].to_csv(train_y_pad_path, index=None, header=False) test_df['X'].to_csv(test_x_pad_path, index=None, header=False) # # print('train_x_max_len:{} ,train_y_max_len:{}'.format(X_max_len, train_y_max_len)) # 11. 词向量再次训练 # print('start retrain w2v model') # wv_model.build_vocab(LineSentence(train_x_pad_path), update=True) # wv_model.train(LineSentence(train_x_pad_path), epochs=1, total_examples=wv_model.corpus_count) # # print('1/3') # wv_model.build_vocab(LineSentence(train_y_pad_path), update=True) # wv_model.train(LineSentence(train_y_pad_path), epochs=1, total_examples=wv_model.corpus_count) # # print('2/3') # wv_model.build_vocab(LineSentence(test_x_pad_path), update=True) # wv_model.train(LineSentence(test_x_pad_path), epochs=1, total_examples=wv_model.corpus_count) # 保存词向量模型 wv_model.save(save_wv_model_path) print('finish retrain w2v model') print('final w2v_model has vocabulary of ', len(wv_model.wv.vocab)) # 12. 更新vocab vocab = {word: index for index, word in enumerate(wv_model.wv.index2word)} reverse_vocab = { index: word for index, word in enumerate(wv_model.wv.index2word) } # 保存字典 save_dict(save_vocab_path, vocab) save_dict(reverse_vocab_path, reverse_vocab) # 13. 保存词向量矩阵 embedding_matrix = wv_model.wv.vectors np.save(embedding_matrix_path, embedding_matrix) # 14. 数据集转换 将词转换成索引 [<START> 方向机 重 ...] -> [32800, 403, 986, 246, 231 # vocab = Vocab() train_ids_x = train_df['X'].apply(lambda x: transform_data(x, vocab)) train_ids_y = train_df['Y'].apply(lambda x: transform_data(x, vocab)) test_ids_x = test_df['X'].apply(lambda x: transform_data(x, vocab)) # 15. 数据转换成numpy数组 # 将索引列表转换成矩阵 [32800, 403, 986, 246, 231] --> array([[32800, 403, 986 ]] train_X = np.array(train_ids_x.tolist()) train_Y = np.array(train_ids_y.tolist()) test_X = np.array(test_ids_x.tolist()) # 保存数据 np.save(train_x_path, train_X) np.save(train_y_path, train_Y) np.save(test_x_path, test_X) return train_X, train_Y, test_X
executor = Parallel(n_jobs=n_jobs, backend="multiprocessing", prefer="processes") do = delayed(partial(tokenize_sentence_corpus, corpus_out_path)) tasks = (do(i, batch) for i, batch in enumerate(partitions)) executor(tasks) # process_texts(documents_path, year='2020', court='01', corpus_out_path=unigram_sentences_path, batch_size=8, n_jobs=2, # debug=True) stop_words = get_custom_stop_words() pruned_words, counters, total_words = Phrases.learn_vocab( sentences=LineSentence(unigram_sentences_path), max_vocab_size=800000000, common_terms=stop_words, progress_per=100) counters = sorted(counters.items(), key=lambda key_value: key_value[1], reverse=True) count = 0 for key, value in counters: count += 1 print(any2unicode(key), value) print(count) bigram_model = Phrases(LineSentence(unigram_sentences_path),
def train_model(corpus_path, outpath): logger.info('Training model...') model = Word2Vec(LineSentence(corpus_path), workers=cpu_count()) model.save(outpath)
# -*-coding=utf-8-*- from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence import codecs filename = 'html.txt' sentences = LineSentence(filename) model = Word2Vec(sentences, size=128, window=5, min_count=5, workers=4) model.save('word_embedding_128') items = model.most_similar('中国') for item in items: print(item[0], item[1]) print(model.similarity('男人', '女人')) filename = 'wikizhword.text' f = codecs.open(filename, 'r', encoding='utf-8') line = 20 for _ in range(line): print(f.readline()) # sentences = LineSentence(f) # model = Word2Vec(sentences,size=128,window=5,min_count=5,workers=4) # model.save('word_embedding_128') # # #model=Word2Evc.load('word_embedding_128') # items = model.most_similar('中国') # for item in items: # print(item[0],item[1]) # #
from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence import jieba import os import logging logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) os.chdir('_normal') lines = [] for file in os.listdir(): lines += LineSentence(file) model = Word2Vec(lines, size=100) # 训练skip-gram模型; 默认window=5 os.chdir('..') model.save('save.model')
import os import codecs from gensim.models import Word2Vec from gensim.models import Phrases from gensim.models.word2vec import LineSentence import pandas as pd import settings trigram_sentences = LineSentence( os.path.join(settings.DATA_PATH, 'trigram_sentences.txt')) word2vec_filepath = os.path.join(settings.DATA_PATH, 'word2vec_model') if 0 == 1: text2vec = Word2Vec(trigram_sentences, size=100, window=5, min_count=20, sg=1, workers=4) text2vec.save(word2vec_filepath) for i in range(1, 12): text2vec.train(trigram_sentences) text2vec.save(word2vec_filepath) text2vec = Word2Vec.load(word2vec_filepath) text2vec.init_sims() print('{} training epochs so far.'.format(text2vec.train_count)) print('{:,} terms in the text2vec vocabulary.'.format(len(text2vec.vocab)))
import logging from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence path = '/Users/chaofeng/Documents/GitHub/advancedTM_Red/data/' trigram_filename = path + 'wikidata-20200213-truthy-BETA.trigrams.bz2' #trigram_filename = 'wikidata-20200213-truthy-BETA.trigrams.bz2' logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) sentences = LineSentence(trigram_filename) filename = path + 'wikidata_cbow300_iter15' w2v = Word2Vec(sentences, size=300, window=1, min_count=20, workers=20, iter=1) w2v.save(filename)
import os import sys root_path = "/home/ubuntu/answerbot-tool/src" sys.path.append(root_path) from gensim.models.word2vec import Word2Vec, LineSentence from utils.time_utils import get_current_time corpus_fpath = '../_1_preprocessing/corpus.txt' print 'start time : ', get_current_time() sentences = LineSentence(corpus_fpath) print "begin training..." # size is the dimensionality of the feature vectors. # window is the maximum distance between the current and predicted word within a sentence. # min_count = ignore all words with total frequency lower than this. # workers = use this many worker threads to train the model (=faster training with multicore machines). model = Word2Vec(sentences, size=200, window=5, min_count=0, workers=4, iter=100) model.save('model') print 'end time : ', get_current_time()
# -*- coding: utf-8 -*- """ Usage: ./train.py <path to processed wiki> <path to output file> Adapted from http://textminingonline.com/training-word2vec-model-on-english-wikipedia-by-gensim """ import logging import os.path import sys from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence if __name__ == '__main__': program = os.path.basename(sys.argv[0]) logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) # check and process input arguments if len(sys.argv) < 3: print globals()['__doc__'] % locals() sys.exit(1) inp, outp1 = sys.argv[1:4] # NOTE: it doesn't shuffle data between passes, which might degrade performance model = Word2Vec(LineSentence(inp), size=300, negative=5, workers=5) model.save_word2vec_format(outp1, binary=False)
#coding:utf-8 from gensim.models import word2vec from gensim.models.word2vec import LineSentence import logging inFile = 'corpus.txt' outFile = 'output_demoModel.out' logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO) model = word2vec.Word2Vec(LineSentence(inFile), size=100, window=3, min_count=1) print model.wv[u'理']
#读入评论内容 all = pd.read_csv('Train_Data.csv', encoding='utf-8') comment = all[all['text'].notnull()] #仅读取非空评论 comment = comment['text'] comment = np.array(comment) for i in range(len(comment)): comment[i] = r.sub('', str(comment[i])) comment = pd.DataFrame(comment) comment['words'] = comment[0] del comment[0] comment['mark'] = all['negative'] comment = comment[3000:] comment['words'] = comment['words'].apply(cw) #评论分词 with open('result.txt', 'w', encoding="utf-8") as f2: f2.write(str(comment['words'])) sentences = LineSentence("D:/PythonCode/yang/test/result.txt") path = get_tmpfile("D:/PythonCode/yang/test/w2v_model.bin") # 创建临时文件 model = Word2Vec(sentences, hs=1, min_count=1, window=10, size=100) # 模型储存与加载1 model.save(path) model = Word2Vec.load("D:/PythonCode/yang/test/w2v_model.bin") # d2v_train = pd.concat([pn['words'], comment['words']], ignore_index = True) # # w = [] #将所有词语整合在一起 # for i in d2v_train: # w.extend(i) # # dict = pd.DataFrame(pd.Series(w).value_counts()) #统计词的出现次数 # del w,d2v_train # dict['id']=list(range(1,len(dict)+1)) #
print("Création d'index ...") #sauvegarder dans un dossier os.mkdir(documentsDict) for f in documents: out_file=open(join(documentsDict,f), 'w') out_file.write(str(documents[f])) out_file.close() out_file=open(doc_freqTerm, 'w') out_file.write(str(doc_freq)) out_file.close() print("corpus traité avec succès \n") #2 apprentissage print("Apprentissage ...") sentences = LineSentence(corpusAsSentences) model = Word2Vec(sentences, size=dimConcept, window=win, min_count=minc, workers=4) # lancer la génération du vocabulaire model.save_word2vec_format(index+'/word2vec'+str(dimConcept)+'_win'+str(win)+'_min'+str(minc)+'.txt', fvocab=None, binary=False) print("vocabulaire ok \n") #3 representation des documents en vecteurs print("Documents to vectors ...") os.mkdir(matDoc) for f in listdir(collection): # ici lire tte la collection doc=documents[f] tdoc=0 # taille du document à calculer : tdoc=sum(doc.values()) vec_doc=numpy.zeros(dimConcept) mat_doc={} for word in doc: if(word in model.vocab):
with open("news_data_8.txt", "w") as text_file: text_file.write(kalimat_panjang) import multiprocessing import logging import os.path import sys import multiprocessing from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence program = os.path.basename(sys.argv[0]) logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s') logging.root.setLevel(level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) namaFileInput = "news_data_8.txt" namaFileOutput = "hoax_en_case" model = Word2Vec(LineSentence(namaFileInput), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count()) # trim unneeded model memory = use (much) less RAM model.init_sims(replace=True) model.save(namaFileOutput)
from gensim.parsing.preprocessing import preprocess_string from gensim.parsing.preprocessing import strip_punctuation from gensim.parsing.preprocessing import strip_multiple_whitespaces from gensim.parsing.preprocessing import strip_non_alphanum from gensim.parsing.preprocessing import remove_stopwords # EXP_HOME = "F:/MyWorks/Thesis Works/Crowdsource_Knowledge_Base/DeepGenQR/experiment" EXP_HOME = "C:/My MSc/ThesisWorks/BigData_Code_Search/DeepGenQR/experiment" csv_file = EXP_HOME + "/stackoverflow/eclipse/eclipse-qa.csv" CUSTOM_FILTERS = [ lambda x: x.lower(), strip_multiple_whitespaces, strip_punctuation, remove_stopwords, strip_non_alphanum ] sentences = LineSentence(open(csv_file, 'r'), max_sentence_length=100000, limit=None) pre_processed = list() for sentence in sentences: # print(' '.join(sentence)) temp = ' '.join(sentence) pp_sentence = preprocess_string(temp, CUSTOM_FILTERS) # print(pp_sentence) pre_processed.append(' '.join(pp_sentence)) # saving the pre-processed to the file myFile = open(pp_raw_code, 'w') for line in pre_processed: myFile.write("%s\n" % line) print("Corpus preprocessed successfully!")
def dict_to_text(): data=read_post_data() text={} for i,t in data.items(): text[i]=split_word(t) return text if __name__ == '__main__': #输入文件:每篇文章转换位1行text文本,并且去掉了标点符号等内容 #中文数据,与英文处理过程相似,也分两个步骤,不过需要对中文数据特殊处理一下,包括繁简转换,中文分词,去除非utf-8字符等。 inp=r'f:\python\data\jiebatext.txt'#输入文件 outp1=r"F:\python\data\text.model"#gensim中默认格式的word2vec model outp2 =r"F:\python\data\text.vector"#原始c版本word2vec的vector格式的模型 model = Word2Vec(LineSentence(inp), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count()) # 参数解释: #1.sg=1是skip-gram算法,对低频词敏感;默认sg=0为CBOW算法。 #2.size是输出词向量的维数,值太小会导致词映射因为冲突而影响结果,值太大则会耗内存并使算法计算变慢,一般值取为100到200之间。 #3.window是句子中当前词与目标词之间的最大距离,3表示在目标词前看3-b个词,后面看b个词(b在0-3之间随机)。 #4.min_count是对词进行过滤,频率小于min-count的单词则会被忽视,默认值为5。 #5.negative和sample可根据训练结果进行微调,sample表示更高频率的词被随机下采样到所设置的阈值,默认值为1e-3。 #6.hs=1表示层级softmax将会被使用,默认hs=0且negative不为0,则负采样将会被选择使用。 #7.workers控制训练的并行,此参数只有在安装了Cpython后才有效,否则只能使用单核。 #word2vec的两种形式:CBOW和Skip-gram模型 #CBOW去除了上下文各词的词序信息,使用上下文各词的平均值。 #skip-gram和CBOW正好相反,它使用单一的焦点词作为输入,经过训练然后输出它的目标上下文 # trim unneeded model memory = use(much) less RAM
from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence import multiprocessing model = Word2Vec( LineSentence('data/simple.reg.txt'), size=400, window=5, min_count=5, workers=multiprocessing.cpu_count() - 2, ) outp1 = 'data/simple.zh.text.model' outp2 = 'data/simple.zh.text.vector' model.save(outp1) model.wv.save_word2vec_format(outp2)